{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NYC Airbnb Price Prediction - Deep Learning model training\n",
    "\n",
    "Use dataset published by Kaggle - https://www.kaggle.com/dgomonov/new-york-city-airbnb-open-data - to train a simple deep learning model to predict prices for Airbnb properties.\n",
    "\n",
    "This notebook contains the code to train the model from the dataset prepared in the airbnb_data_preparation notebook. \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Links to key parts of the notebook <a name='linkanchor' />\n",
    "<a href=#ingestdash>Ingest data</a>\n",
    "\n",
    "<a href=#definecategories>Define feature categories</a>\n",
    "\n",
    "<a href=#modelfit>Define and fit model</a>\n",
    "\n",
    "<a href=#reload>Reload saved model and weights</a>\n",
    "\n",
    "<a href=#confusionmatrix>Confusion matrix</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Common imports and global variable definitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# common imports\n",
    "import zipfile\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "import seaborn as sns\n",
    "# import datetime, timedelta\n",
    "import datetime\n",
    "import pydotplus\n",
    "from datetime import datetime, timedelta\n",
    "from datetime import date\n",
    "from dateutil import relativedelta\n",
    "from io import StringIO\n",
    "import pandas as pd\n",
    "import pickle\n",
    "from pickle import dump\n",
    "from pickle import load\n",
    "from sklearn.base import BaseEstimator\n",
    "from sklearn.base import TransformerMixin\n",
    "# DSX code to import uploaded documents\n",
    "from io import StringIO\n",
    "import requests\n",
    "import json\n",
    "from sklearn.preprocessing import LabelEncoder, MinMaxScaler, StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import os\n",
    "import yaml\n",
    "import math\n",
    "import sys\n",
    "from subprocess import check_output\n",
    "from IPython.display import display\n",
    "#model libraries\n",
    "from tensorflow.keras.metrics import Accuracy, Recall, Precision\n",
    "from tensorflow.keras.layers import Input, Dropout, Dense, BatchNormalization, Activation, concatenate, GRU, Embedding, Flatten, BatchNormalization\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, Callback, EarlyStopping\n",
    "from tensorflow.keras import regularizers\n",
    "from tensorflow.keras.layers import BatchNormalization\n",
    "#from tf.keras.layers.normalization import BatchNormalization\n",
    "from tensorflow.keras.regularizers import l2\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.optimizers import SGD\n",
    "from tensorflow.keras import backend as K\n",
    "# from tensorflow.keras.utils.vis_utils import plot_model\n",
    "from tensorflow.keras.utils import plot_model\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.models import load_model\n",
    "from tensorflow.python.keras.callbacks import TensorBoard\n",
    "#import datetime\n",
    "#from datetime import date\n",
    "from sklearn import metrics\n",
    "# import pipeline libraries\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.base import TransformerMixin\n",
    "from sklearn.base import BaseEstimator\n",
    "from custom_classes import encode_categorical\n",
    "from custom_classes import prep_for_keras_input\n",
    "from custom_classes import fill_empty\n",
    "from custom_classes import encode_text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.0.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "tf.__version__ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current directory is: C:\\personal\\manning\\second_example_june_2020\\airbnb\\dl_example_2\\notebooks\n",
      "path_to_yaml C:\\personal\\manning\\second_example_june_2020\\airbnb\\dl_example_2\\notebooks\\airbnb_model_training_config.yml\n"
     ]
    }
   ],
   "source": [
    "# load config file\n",
    "current_path = os.getcwd()\n",
    "print(\"current directory is: \"+current_path)\n",
    "\n",
    "path_to_yaml = os.path.join(current_path, 'airbnb_model_training_config.yml')\n",
    "print(\"path_to_yaml \"+path_to_yaml)\n",
    "try:\n",
    "    with open (path_to_yaml, 'r') as c_file:\n",
    "        config = yaml.safe_load(c_file)\n",
    "except Exception as e:\n",
    "    print('Error reading the config file')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "date today 2020-06-26 21:48:55.287926\n",
      "start date 2014-01-01\n",
      "end date 2019-12-31\n"
     ]
    }
   ],
   "source": [
    "# load parameters\n",
    "\n",
    "repeatable_run = config['test_parms']['repeatable_run']\n",
    "# fix seeds to get identical results on mulitiple runs\n",
    "if repeatable_run:\n",
    "    from numpy.random import seed\n",
    "    seed(4)\n",
    "    tf.random.set_seed(7)\n",
    "\n",
    "\n",
    "testproportion = config['test_parms']['testproportion'] # proportion of data reserved for test set\n",
    "trainproportion = config['test_parms']['trainproportion'] # proportion of non-test data dedicated to training (vs. validation)\n",
    "get_test_train_acc = config['test_parms']['get_test_train_acc']\n",
    "verboseout = config['general']['verboseout']\n",
    "includetext = config['general']['includetext'] # switch to determine whether text columns are included in the model\n",
    "save_model_plot = config['general']['save_model_plot'] # switch to determine whether to generate plot with plot_model\n",
    "tensorboard_callback = config['general']['tensorboard_callback'] # switch to determine if tensorboard callback defined\n",
    "\n",
    "presaved = config['general']['presaved']\n",
    "savemodel = config['general']['savemodel']\n",
    "picklemodel = config['general']['picklemodel']\n",
    "hctextmax = config['general']['hctextmax']\n",
    "maxwords = config['general']['maxwords']\n",
    "textmax = config['general']['textmax']\n",
    "\n",
    "targetthresh = config['general']['targetthresh']\n",
    "targetcontinuous = config['general']['targetcontinuous']\n",
    "\n",
    "#time of day thresholds\n",
    "time_of_day = {'overnight':{'start':0,'end':5},'morning_rush':{'start':5,'end':10},\n",
    "              'midday':{'start':10,'end':15},'aft_rush':{'start':15,'end':19},'evening':{'start':19,'end':24}}\n",
    "\n",
    "\n",
    "\n",
    "emptythresh = config['general']['emptythresh']\n",
    "zero_weight = config['general']['zero_weight']\n",
    "one_weight = config['general']['one_weight']\n",
    "one_weight_offset = config['general']['one_weight_offset']\n",
    "patience_threshold = config['general']['patience_threshold']\n",
    "\n",
    "\n",
    "# modifier for saved model elements\n",
    "modifier = config['general']['modifier']\n",
    "\n",
    "# control whether training controlled by early stop\n",
    "early_stop = True\n",
    "\n",
    "# default hyperparameter values\n",
    "learning_rate = config['hyperparameters']['learning_rate']\n",
    "dropout_rate = config['hyperparameters']['dropout_rate']\n",
    "l2_lambda = config['hyperparameters']['l2_lambda']\n",
    "loss_func = config['hyperparameters']['loss_func']\n",
    "output_activation = config['hyperparameters']['output_activation']\n",
    "batch_size = config['hyperparameters']['batch_size']\n",
    "epochs = config['hyperparameters']['epochs']\n",
    "\n",
    "# date values\n",
    "date_today = datetime.now()\n",
    "print(\"date today\",date_today)\n",
    "start_date =  date(config['general']['start_year'],config['general']['start_month'], config['general']['start_day'])\n",
    "print(\"start date\",start_date)\n",
    "end_date = date(config['general']['end_year'],config['general']['end_month'], config['general']['end_day'])\n",
    "print(\"end date\",end_date)\n",
    "\n",
    "\n",
    "# pickled original dataset and post-preprocessing dataset\n",
    "pickled_data_file = config['general']['pickled_data_file']\n",
    "pickled_dataframe = config['general']['pickled_dataframe']\n",
    "routedirection_file = config['general']['route_direction_file']\n",
    "\n",
    "# experiment parameter\n",
    "\n",
    "current_experiment = config['test_parms']['current_experiment']\n",
    "\n",
    "# load lists of column categories\n",
    "collist = config['categorical']\n",
    "textcols = config['text']\n",
    "continuouscols = config['continuous']\n",
    "excludefromcolist = config['excluded']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# time_of_day = {'overnight':{'start':0,'end':5},'morning_rush':{'start':5,'end':10},\n",
    "#              'midday':{'start':10,'end':15},'aft_rush':{'start':15,'end':19},'evening':{'start':19,'end':23}}\n",
    "\n",
    "\n",
    "def get_time(hour):\n",
    "    for tod in time_of_day:\n",
    "        if (hour >= time_of_day[tod]['start']) and (hour < time_of_day[tod]['end']):\n",
    "            tod_out = tod\n",
    "    return(tod_out)\n",
    "\n",
    "def weekend_time(day, tod):\n",
    "    if (day=='Saturday') or (day=='Sunday'):\n",
    "        return('w'+tod)\n",
    "    else:\n",
    "        return(tod)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the paths required\n",
    "\n",
    "def get_path():\n",
    "    '''get the path for data files'''\n",
    "    rawpath = os.getcwd()\n",
    "    # data is in a directory called \"data\" that is a sibling to the directory containing the notebook\n",
    "    path = os.path.abspath(os.path.join(rawpath, '..', 'data'))\n",
    "    return(path)\n",
    "\n",
    "def get_pipeline_path():\n",
    "    '''get the path for data files'''\n",
    "    rawpath = os.getcwd()\n",
    "    # data is in a directory called \"data\" that is a sibling to the directory containing the notebook\n",
    "    path = os.path.abspath(os.path.join(rawpath, '..', 'pipelines'))\n",
    "    return(path)\n",
    "\n",
    "def get_model_path():\n",
    "    '''get the path for data files'''\n",
    "    rawpath = os.getcwd()\n",
    "    # data is in a directory called \"data\" that is a sibling to the directory containing the notebook\n",
    "    path = os.path.abspath(os.path.join(rawpath, '..', 'models'))\n",
    "    return(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_experiment_parameters(experiment_number, count_no_delay, count_delay):\n",
    "    ''' set the appropriate parameters for the experiment '''\n",
    "    print(\"setting parameters for experiment \", experiment_number)\n",
    "    # default settings for early stopping:\n",
    "    es_monitor = \"val_loss\"\n",
    "    es_mode = \"min\"\n",
    "    if experiment_number == 0:\n",
    "        #\n",
    "        early_stop = False\n",
    "        #\n",
    "        one_weight = 1.0\n",
    "        #\n",
    "        epochs = 1\n",
    "    elif experiment_number == 9:\n",
    "        #\n",
    "        early_stop = True\n",
    "        es_monitor=\"val_accuracy\"\n",
    "        es_mode = \"max\"\n",
    "        #\n",
    "        one_weight = (count_no_delay/count_delay) + one_weight_offset\n",
    "        #\n",
    "        get_test_train_acc = False\n",
    "        #\n",
    "        epochs = 20    \n",
    "    elif experiment_number == 1:\n",
    "        #\n",
    "        early_stop = False\n",
    "        #\n",
    "        one_weight = 1.0\n",
    "        #\n",
    "        epochs = 10\n",
    "    elif experiment_number == 2:\n",
    "        #\n",
    "        early_stop = False\n",
    "        #\n",
    "        one_weight = 1.0\n",
    "        #\n",
    "        epochs = 50\n",
    "    elif experiment_number == 3:\n",
    "        #\n",
    "        early_stop = False\n",
    "        #\n",
    "        one_weight = (count_no_delay/count_delay) + one_weight_offset\n",
    "        #\n",
    "        epochs = 50\n",
    "    elif experiment_number == 4:\n",
    "        #\n",
    "        early_stop = True\n",
    "        es_monitor = \"val_loss\"\n",
    "        es_mode = \"min\"\n",
    "        #\n",
    "        one_weight = (count_no_delay/count_delay) + one_weight_offset\n",
    "        #\n",
    "        epochs = 50\n",
    "    elif experiment_number == 5:\n",
    "        #\n",
    "        early_stop = True\n",
    "        # if early stopping fails because the level of TensorFlow/Python, comment out the following\n",
    "        # line and uncomment the subsequent if statement\n",
    "        es_monitor=\"val_accuracy\"\n",
    "        '''\n",
    "        if sys.version_info >= (3,7):\n",
    "            es_monitor=\"val_accuracy\"\n",
    "        else:\n",
    "            es_monitor = \"val_acc\"\n",
    "        '''\n",
    "        es_mode = \"max\"\n",
    "        #\n",
    "        one_weight = (count_no_delay/count_delay) + one_weight_offset\n",
    "        #\n",
    "        epochs = 50\n",
    "    else:\n",
    "        early_stop = True\n",
    "    return(early_stop, one_weight, epochs,es_monitor,es_mode)\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ingest data and create refactored dataframe\n",
    "- Ingest data for route information and delay information\n",
    "- Create refactored dataframe with one row per route / direction / timeslot combination\n",
    "\n",
    "<a name='ingestdash' />\n",
    "<a href=#linkanchor>Back to link list</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load list of valid routes and directions into dataframe\n",
    "def ingest_data(path):\n",
    "    file_name = os.path.join(path,pickled_dataframe)\n",
    "    merged_data = pd.read_pickle(file_name)\n",
    "    merged_data.head()\n",
    "    return(merged_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add derived columns to merged_data dataframe\n",
    "def prep_merged_data(merged_data):\n",
    "    '''\n",
    "    # define cols for year month day hour\n",
    "    merged_data['year'] = pd.DatetimeIndex(merged_data['Report Date']).year\n",
    "    merged_data['month'] = pd.DatetimeIndex(merged_data['Report Date']).month\n",
    "    merged_data['daym'] = pd.DatetimeIndex(merged_data['Report Date']).day\n",
    "    merged_data['hour'] = pd.DatetimeIndex(merged_data['Report Date Time']).hour\n",
    "    # define time of day column\n",
    "    merged_data['time_of_day'] = merged_data['hour'].apply(lambda x:get_time(x))\n",
    "    # add a special timeframe for weekends\n",
    "    merged_data['time_of_day'] = merged_data.apply(lambda x: weekend_time(x['Day'], x['time_of_day']), axis=1)\n",
    "    '''\n",
    "    if targetcontinuous:\n",
    "        merged_data['target'] = merged_data['price']\n",
    "    else:\n",
    "        merged_data['target'] = np.where(merged_data['price'] >= merged_data['price'].mean(), 1, 0 )\n",
    "    return(merged_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate dataframe containing rows for each timeslot for each route for each direction\n",
    "# and merge with the input merged_data dataframe to get a result of a sparse dataframe with the\n",
    "# timeslot / route / direction combinations where delays occurred\n",
    "def prep_sparse_df(routedirection_frame, merged_data):\n",
    "    routedirection_frame['count'] = 0\n",
    "    print(\"routedirection\")\n",
    "    display(routedirection_frame[:5])\n",
    "    # define a dataframe with a row for each date to be covered\n",
    "    days = pd.date_range(start_date, end_date, freq='D')\n",
    "    date_frame = pd.DataFrame({'date':days,'count':0})\n",
    "    print(\"date_frame\")\n",
    "    display(date_frame[:5])\n",
    "    # define a dataframe with a row for each hour\n",
    "    hour_list = list(range(0,24))\n",
    "    hour_frame = pd.DataFrame({'hour':hour_list,'count':0})\n",
    "    print(\"hour_frame\")\n",
    "    display(hour_frame[:5])\n",
    "    #vprint(hour_frame.head())\n",
    "    # merge date_frame and routedirection\n",
    "    result1 = pd.merge(date_frame, routedirection_frame, on='count', how='outer')\n",
    "    print(\"result1\")\n",
    "    display(result1[:5])\n",
    "    # merge result1 with hour_frame\n",
    "    result2 = pd.merge(result1, hour_frame, on='count', how='outer')\n",
    "    result2 = result2.rename(columns={'date': 'Report Date'})\n",
    "    result2.Route = result2.Route.astype(str)\n",
    "    # segment the date\n",
    "    result2['year'] = pd.DatetimeIndex(result2['Report Date']).year\n",
    "    result2['month'] = pd.DatetimeIndex(result2['Report Date']).month\n",
    "    result2['daym'] = pd.DatetimeIndex(result2['Report Date']).day\n",
    "    result2['day'] = pd.DatetimeIndex(result2['Report Date']).weekday\n",
    "    print(\"result2\")\n",
    "    display(result2[:5])\n",
    "    print(\"merged_data before\")\n",
    "    display(merged_data[:5])\n",
    "    # drop extraneous columns from merged_data\n",
    "    merged_data = merged_data.drop(['Time',\n",
    "     'Report Date Time',\n",
    "     'year',\n",
    "     'month',\n",
    "     'daym',\n",
    "     'time_of_day','Min Gap','Location','Incident','Vehicle','target','Day'],axis=1)\n",
    "    print(\"merged_data after dropping extraneous columns\")\n",
    "    display(merged_data[:5])\n",
    "    # join result2 and the trimmed merged_data\n",
    "    result3 = pd.merge(result2,merged_data ,how='left', on=['Report Date','Route','Direction','hour'])\n",
    "    result3['Min Delay'].fillna(value=0.0,inplace=True)\n",
    "    result3['target'] = np.where(result3['Min Delay'] > 0.0, 1, 0 )\n",
    "    print(\"result3\")\n",
    "    display(result3[:5])\n",
    "    return(result3)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Master Prep Calls\n",
    "Contains calls to functions to load data, prep input dataframes, and create refactored dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "path is C:\\personal\\manning\\second_example_june_2020\\airbnb\\dl_example_2\\data\n",
      "shape of pre refactored dataset (48895, 17)\n",
      "shape of refactored dataset (48895, 17)\n",
      "count under mean  34016\n",
      "count over mean  14879\n",
      "setting parameters for experiment  1\n",
      "early_stop is  False\n",
      "one_weight is  1.0\n",
      "epochs is  10\n",
      "es_monitor is  val_loss\n",
      "es_mode is  min\n"
     ]
    }
   ],
   "source": [
    "# master calls\n",
    "# get the path for data files\n",
    "path = get_path()\n",
    "print(\"path is\",path)\n",
    "# load route direction and delay data datframes\n",
    "merged_data = ingest_data(path)\n",
    "merged_data = prep_merged_data(merged_data)\n",
    "print(\"shape of pre refactored dataset\", merged_data.shape)\n",
    "#merged_data['year'].value_counts()\n",
    "#merged_data.groupby(['Route','Direction']).size().reset_index().rename(columns={0:'count'}).tail(50)\n",
    "# create refactored dataframe with one row for each route / direction / timeslot combination\n",
    "#merged_data = prep_sparse_df(directions_df, merged_data)\n",
    "print(\"shape of refactored dataset\", merged_data.shape)\n",
    "count_no_delay = merged_data[merged_data['target']==0].shape[0]\n",
    "count_delay = merged_data[merged_data['target']==1].shape[0]\n",
    "print(\"count under mean \",count_no_delay)\n",
    "print(\"count over mean \",count_delay)\n",
    "# define parameters for the current experiment\n",
    "experiment_number = current_experiment\n",
    "early_stop, one_weight, epochs,es_monitor,es_mode = set_experiment_parameters(experiment_number, count_no_delay, count_delay)\n",
    "print(\"early_stop is \",early_stop)\n",
    "print(\"one_weight is \",one_weight)\n",
    "print(\"epochs is \",epochs)\n",
    "print(\"es_monitor is \",es_monitor)\n",
    "print(\"es_mode is \",es_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(48895, 17)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define test / training sets; encode categorical values; process text field\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get training and test data set\n",
    "def get_train_validation_test(dataset):\n",
    "    train, test = train_test_split(dataset, test_size = testproportion)\n",
    "    dtrain, dvalid = train_test_split(train, random_state=123, train_size=trainproportion)\n",
    "    print(\"Through train test split. Test proportion:\")\n",
    "    print(testproportion)\n",
    "    return(dtrain,dvalid,test)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define feature categories <a name='definecategories' />\n",
    "<a href=#linkanchor>Back to link list</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all cols ['id', 'name', 'host_id', 'host_name', 'neighbourhood_group', 'neighbourhood', 'latitude', 'longitude', 'room_type', 'price', 'minimum_nights', 'number_of_reviews', 'last_review', 'reviews_per_month', 'calculated_host_listings_count', 'availability_365', 'target']\n"
     ]
    }
   ],
   "source": [
    "allcols = list(merged_data)\n",
    "print(\"all cols\",allcols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the required column lists\n",
    "\n",
    "\n",
    "def def_col_lists():\n",
    "    allcols = list(merged_data)\n",
    "    textcols = [] # columns to deal with as text - replace entries with multiple IDs and use embeddings, RNN\n",
    "    continuouscols = ['minimum_nights','number_of_reviews','reviews_per_month','calculated_host_listings_count'] # columns to deal with as continuous values - no embeddings\n",
    "    if targetcontinuous:\n",
    "        excludefromcolist = ['price','id','latitude','longitude','name','host_name','last_review'] # columns to exclude completely from the model\n",
    "\n",
    "    else:\n",
    "        # if target column is not renamed Min Delay put Min Delay in exclusion list\n",
    "        excludefromcolist = ['price','id','latitude','longitude','name','host_name','last_review'] # columns to exclude completely from the model\n",
    "    nontextcols = list(set(allcols) - set(textcols))\n",
    "    collist = list(set(nontextcols) - set(excludefromcolist) - set(continuouscols))\n",
    "    for col in continuouscols:\n",
    "        print(\"col is\",col)\n",
    "        merged_data[col] = merged_data[col].astype(float)\n",
    "        print(\"got through one\")\n",
    "    # print column list lengths and contents:\n",
    "    print(\"allcols\",len(allcols))\n",
    "    print(\"excludefromcolist\",len(excludefromcolist))\n",
    "    print(excludefromcolist)\n",
    "    print(\"textcols\",len(textcols))\n",
    "    print(textcols)\n",
    "    print(\"continuouscols\",len(continuouscols))\n",
    "    print(continuouscols)\n",
    "    print(\"collist\",len(collist))\n",
    "    print(collist)\n",
    "    return(collist,continuouscols,textcols)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Invoke Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define keras variables\n",
    "\n",
    "\n",
    "# X for the features used\n",
    "\n",
    "def get_keras_vars(dataset):\n",
    "    X = {}\n",
    "    dictlist = []\n",
    "    for col in collist:\n",
    "        if verboseout:\n",
    "            print(\"cat col is\",col)\n",
    "        X[col] = np.array(dataset[col])\n",
    "        dictlist.append(np.array(dataset[col]))\n",
    "\n",
    "    for col in textcols:\n",
    "        if verboseout:\n",
    "            print(\"text col is\",col)\n",
    "        X[col] = pad_sequences(dataset[col], maxlen=max_dict[col])\n",
    "        dictlist.append(pad_sequences(dataset[col], maxlen=max_dict[col]))\n",
    "\n",
    "    for col in continuouscols:\n",
    "        if verboseout:\n",
    "            print(\"cont col is\",col)\n",
    "        X[col] = np.array(dataset[col])\n",
    "        dictlist.append(np.array(dataset[col]))\n",
    "\n",
    "    return X, dictlist\n",
    "\n",
    "def get_keras_list_only(X_in):\n",
    "    dictlist = []\n",
    "    for key, value in X_in.items():\n",
    "        print(\"X def loop key\",key)\n",
    "        print(\"value shape\",value.shape)\n",
    "        temp = [key,value]\n",
    "        dictlist.append(value)\n",
    "    return dictlist\n",
    "\n",
    "def get_keras_np(X_in):\n",
    "    return np.array(list(X_in.items()),dtype=object)\n",
    "# np.array(list(result.items()), dtype=dtype)\n",
    "\n",
    "# the deployment API for Watson Studio can only take a list/array, not a dictionary, so define list-only version for input\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "collist is:  ['neighbourhood_group', 'neighbourhood', 'room_type']\n",
      "textcols is:  []\n",
      "continuouscols is:  ['minimum_nights', 'number_of_reviews', 'reviews_per_month', 'calculated_host_listings_count']\n",
      "excludefromcolist is:  ['price', 'id', 'latitude', 'longitude', 'host_name', 'last_review', 'name', 'host_name', 'availability_365']\n"
     ]
    }
   ],
   "source": [
    "print(\"collist is: \",str(collist))\n",
    "print(\"textcols is: \",str(textcols))\n",
    "print(\"continuouscols is: \",str(continuouscols))\n",
    "print(\"excludefromcolist is: \",str(excludefromcolist))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "col is minimum_nights\n",
      "got through one\n",
      "col is number_of_reviews\n",
      "got through one\n",
      "col is reviews_per_month\n",
      "got through one\n",
      "col is calculated_host_listings_count\n",
      "got through one\n",
      "allcols 17\n",
      "excludefromcolist 7\n",
      "['price', 'id', 'latitude', 'longitude', 'name', 'host_name', 'last_review']\n",
      "textcols 0\n",
      "[]\n",
      "continuouscols 4\n",
      "['minimum_nights', 'number_of_reviews', 'reviews_per_month', 'calculated_host_listings_count']\n",
      "collist 6\n",
      "['target', 'room_type', 'neighbourhood', 'availability_365', 'host_id', 'neighbourhood_group']\n",
      "fill empty xform\n",
      "col is  target\n",
      "col is  room_type\n",
      "col is  neighbourhood\n",
      "col is  availability_365\n",
      "col is  host_id\n",
      "col is  neighbourhood_group\n",
      "transform col is  target\n",
      "after transform col is  target\n",
      "transform col is  room_type\n",
      "after transform col is  room_type\n",
      "transform col is  neighbourhood\n",
      "after transform col is  neighbourhood\n",
      "transform col is  availability_365\n",
      "after transform col is  availability_365\n",
      "transform col is  host_id\n",
      "after transform col is  host_id\n",
      "transform col is  neighbourhood_group\n",
      "after transform col is  neighbourhood_group\n",
      "Through train test split. Test proportion:\n",
      "0.2\n",
      "cat col is target\n",
      "cat col is room_type\n",
      "cat col is neighbourhood\n",
      "cat col is availability_365\n",
      "cat col is host_id\n",
      "cat col is neighbourhood_group\n",
      "cont col is minimum_nights\n",
      "cont col is number_of_reviews\n",
      "cont col is reviews_per_month\n",
      "cont col is calculated_host_listings_count\n",
      "cat col is target\n",
      "cat col is room_type\n",
      "cat col is neighbourhood\n",
      "cat col is availability_365\n",
      "cat col is host_id\n",
      "cat col is neighbourhood_group\n",
      "cont col is minimum_nights\n",
      "cont col is number_of_reviews\n",
      "cont col is reviews_per_month\n",
      "cont col is calculated_host_listings_count\n",
      "cat col is target\n",
      "cat col is room_type\n",
      "cat col is neighbourhood\n",
      "cat col is availability_365\n",
      "cat col is host_id\n",
      "cat col is neighbourhood_group\n",
      "cont col is minimum_nights\n",
      "cont col is number_of_reviews\n",
      "cont col is reviews_per_month\n",
      "cont col is calculated_host_listings_count\n",
      "keras variables defined\n",
      "X_train_list [array([0, 0, 0, ..., 0, 1, 0], dtype=int64), array([1, 1, 1, ..., 1, 0, 1]), array([ 50,  94,  28, ..., 201, 214, 117]), array([306,  54, 281, ...,  33,   0,   0], dtype=int64), array([34460, 36850, 28945, ...,  9488,  3416, 33503], dtype=int64), array([3, 2, 1, ..., 2, 1, 3]), array([2., 1., 2., ..., 7., 3., 2.]), array([ 5.,  2., 12., ...,  1., 25., 21.]), array([1.15, 1.71, 0.44, ..., 0.11, 0.29, 3.25]), array([6., 3., 2., ..., 2., 1., 2.])]\n"
     ]
    }
   ],
   "source": [
    "# master block to invoke pipeline\n",
    "\n",
    "# build fully qualified names for the files for saving the pipelines\n",
    "pipeline_path = get_pipeline_path()\n",
    "pipeline1_file_name = os.path.join(pipeline_path,'sc_delay_pipleline'+modifier+'.pkl')\n",
    "pipeline2_file_name = os.path.join(pipeline_path,'sc_delay_pipleline_keras_prep'+modifier+'.pkl')\n",
    "\n",
    "# define column lists:\n",
    "collist,continuouscols,textcols = def_col_lists()\n",
    "\n",
    "# create objects of the pipeline classes\n",
    "fe = fill_empty()\n",
    "ec = encode_categorical()\n",
    "pk = prep_for_keras_input()\n",
    "\n",
    "# need to implement the pipeline in two parts:\n",
    "# 1. fill empty + encode categoricals\n",
    "# 2. prep for Keras\n",
    "# because part 1 needs to be applied to the entire dataset and part 2 to the individual train, validate, and test sets\n",
    "\n",
    "\n",
    "sc_delay_pipeline = Pipeline([('fill_empty',fe),('encode_categorical',ec)])\n",
    "sc_delay_pipeline_keras_prep = Pipeline([('prep_for_keras',pk)])\n",
    "\n",
    "\n",
    "\n",
    "# provide the value for each parameter of each of the pipeline classes\n",
    "\n",
    "sc_delay_pipeline.set_params(fill_empty__collist = collist, fill_empty__continuouscols = continuouscols,\n",
    "                            fill_empty__textcols = textcols,encode_categorical__col_list = collist)\n",
    "sc_delay_pipeline_keras_prep.set_params(prep_for_keras__collist = collist,\n",
    "                            prep_for_keras__continuouscols = continuouscols,\n",
    "                            prep_for_keras__textcols = textcols)\n",
    "\n",
    "# fit the input dataset to the pipeline\n",
    "\n",
    "# first fit the first segment of pipeline on the whole dataset\n",
    "X = sc_delay_pipeline.fit_transform(merged_data)\n",
    "max_dict = ec.max_dict\n",
    "# then split dataset\n",
    "dump(sc_delay_pipeline, open(pipeline1_file_name,'wb'))\n",
    "dump(sc_delay_pipeline_keras_prep, open(pipeline2_file_name,'wb'))\n",
    "dtrain, dvalid, test = get_train_validation_test(X)\n",
    "# then apply second portion of pipeline to each subset\n",
    "\n",
    "X_train, X_train_list = get_keras_vars(dtrain)\n",
    "X_valid, X_valid_list = get_keras_vars(dvalid)\n",
    "X_test,X_test_list = get_keras_vars(test)\n",
    "\n",
    "print(\"keras variables defined\")\n",
    "print(\"X_train_list\",X_train_list)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define and fit model <a name='modelfit' />\n",
    "<a href=#linkanchor>Back to link list</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "about to define embeddings\n",
      "textmax is 50\n",
      "through loops for cols\n",
      "through definition of non-text parts of main_l\n",
      "main_l Tensor(\"concatenate_8/Identity:0\", shape=(None, 64), dtype=float32)\n",
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "target (InputLayer)             [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "room_type (InputLayer)          [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 1, 10)        20          target[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 1, 10)        30          room_type[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "neighbourhood (InputLayer)      [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization (BatchNorma (None, 1, 10)        40          embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 1, 10)        40          embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, 1, 10)        2210        neighbourhood[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "availability_365 (InputLayer)   [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 10)           0           batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 10)           0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 1, 10)        40          embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_3 (Embedding)         (None, 1, 10)        3660        availability_365[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "host_id (InputLayer)            [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 10)           0           flatten[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 10)           0           flatten_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_2 (Flatten)             (None, 10)           0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 1, 10)        40          embedding_3[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_4 (Embedding)         (None, 1, 10)        374570      host_id[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "neighbourhood_group (InputLayer [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 20)           0           dropout[0][0]                    \n",
      "                                                                 dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 10)           0           flatten_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_3 (Flatten)             (None, 10)           0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 1, 10)        40          embedding_4[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_5 (Embedding)         (None, 1, 10)        50          neighbourhood_group[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 30)           0           concatenate[0][0]                \n",
      "                                                                 dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 10)           0           flatten_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_4 (Flatten)             (None, 10)           0           batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 1, 10)        40          embedding_5[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 40)           0           concatenate_1[0][0]              \n",
      "                                                                 dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 10)           0           flatten_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_5 (Flatten)             (None, 10)           0           batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 50)           0           concatenate_2[0][0]              \n",
      "                                                                 dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 10)           0           flatten_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_4 (Concatenate)     (None, 60)           0           concatenate_3[0][0]              \n",
      "                                                                 dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "minimum_nights (InputLayer)     [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_5 (Concatenate)     (None, 61)           0           concatenate_4[0][0]              \n",
      "                                                                 minimum_nights[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "number_of_reviews (InputLayer)  [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_6 (Concatenate)     (None, 62)           0           concatenate_5[0][0]              \n",
      "                                                                 number_of_reviews[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "reviews_per_month (InputLayer)  [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_7 (Concatenate)     (None, 63)           0           concatenate_6[0][0]              \n",
      "                                                                 reviews_per_month[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "calculated_host_listings_count  [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_8 (Concatenate)     (None, 64)           0           concatenate_7[0][0]              \n",
      "                                                                 calculated_host_listings_count[0]\n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 1)            65          concatenate_8[0][0]              \n",
      "==================================================================================================\n",
      "Total params: 380,845\n",
      "Trainable params: 380,725\n",
      "Non-trainable params: 120\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# define model in Keras\n",
    "\n",
    "def get_model():\n",
    "\n",
    "\n",
    "    catinputs = {} # list of categorical inputs\n",
    "    textinputs = {} # list of text inputs\n",
    "    continputs = {} # list of continuous inputs\n",
    "    embeddings = {}\n",
    "    textembeddings = {}\n",
    "    catemb = 10 # size of categorical embeddings\n",
    "    textemb = 50 # size of text embeddings\n",
    "\n",
    "\n",
    "    print(\"about to define embeddings\")\n",
    "    collistfix = []\n",
    "    textlayerlist = []\n",
    "    inputlayerlist = []\n",
    "    i = 0\n",
    "    print(\"textmax is\",textmax)\n",
    "    # define layers for categorical columns\n",
    "    for col in collist:\n",
    "        catinputs[col] = Input(shape=[1],name=col)\n",
    "        inputlayerlist.append(catinputs[col])\n",
    "        #print(\"inputname\",inputname)\n",
    "        embeddings[col] = (Embedding(max_dict[col],catemb) (catinputs[col]))\n",
    "        # batchnorm all\n",
    "        embeddings[col] = (BatchNormalization() (embeddings[col]))\n",
    "        collistfix.append(embeddings[col])\n",
    "\n",
    "\n",
    "\n",
    "    # define layers for text columns\n",
    "    if includetext:\n",
    "        for col in textcols:\n",
    "            print(\"col\",col)\n",
    "            textinputs[col] = Input(shape=[X_train[col].shape[1]], name=col)\n",
    "            print(\"text input shape\",X_train[col].shape[1])\n",
    "            inputlayerlist.append(textinputs[col])\n",
    "            textembeddings[col] = (Embedding(textmax,textemb) (textinputs[col]))\n",
    "            textembeddings[col] = (BatchNormalization() (textembeddings[col]))\n",
    "            textembeddings[col] = Dropout(dropout_rate) ( GRU(16,kernel_regularizer=l2(l2_lambda)) (textembeddings[col]))\n",
    "            collistfix.append(textembeddings[col])\n",
    "            print(\"max in the midst\",np.max([np.max(train[col].max()), np.max(test[col].max())])+10)\n",
    "        print(\"through loops for cols\")\n",
    "\n",
    "    # define layers for continuous columns\n",
    "    for col in continuouscols:\n",
    "        continputs[col] = Input(shape=[1],name=col)\n",
    "        inputlayerlist.append(continputs[col])\n",
    "\n",
    "\n",
    "\n",
    "    # build up layers\n",
    "    # main_l = concatenate([Dropout(dropout_rate) (Flatten() (embeddings['Vehicle']) ),Dropout(dropout_rate) (Flatten() (embeddings['Direction']) )])\n",
    "    main_l = concatenate([Dropout(dropout_rate) (Flatten() (embeddings[collist[0]]) ),Dropout(dropout_rate) (Flatten() (embeddings[collist[1]]) )])\n",
    "    for cols in collist:\n",
    "        if (cols != collist[0]) & (cols != collist[1]):\n",
    "            main_l = concatenate([main_l,Dropout(dropout_rate) (Flatten() (embeddings[cols]) )])\n",
    "\n",
    "    print(\"through definition of non-text parts of main_l\")\n",
    "    if includetext:\n",
    "        for col in textcols:\n",
    "            main_l = concatenate([main_l,textembeddings[col]])\n",
    "\n",
    "    for col in continuouscols:\n",
    "        main_l = concatenate([main_l,continputs[col]])\n",
    "\n",
    "    print(\"main_l\", main_l)\n",
    "\n",
    "\n",
    "\n",
    "    # define output layer\n",
    "    output = Dense(1, activation=output_activation) (main_l)\n",
    "\n",
    "    # define model\n",
    "\n",
    "    model = Model(inputlayerlist, output)\n",
    "\n",
    "\n",
    "    # define optimizer\n",
    "    optimizer = SGD(lr=learning_rate)\n",
    "\n",
    "    # compile model\n",
    "    model.compile(loss=loss_func, optimizer=optimizer, metrics=[\"accuracy\"], weighted_metrics=[\"accuracy\"])\n",
    "    # model.compile(loss=loss_func, optimizer=optimizer, metrics=[\"accuracy\", Recall(), Precision()], weighted_metrics=[\"accuracy\"])\n",
    "    \n",
    "\n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "model = get_model()\n",
    "\n",
    "# output model summary\n",
    "\n",
    "model.summary()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save model plot\n",
    "if save_model_plot:\n",
    "    model_plot_file = \"model_plot\"+modifier+\".png\"\n",
    "    model_plot_path = os.path.join(get_path(),model_plot_file)\n",
    "    print(\"model plot path: \",model_plot_path)\n",
    "    plot_model(model, to_file=model_plot_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set up early stopping\n",
    "def set_early_stop(es_monitor, es_mode):\n",
    "    ''' given monitoring parameter es_monitor and mode es_mode, define early stopping callback, save model callback, and \n",
    "    TensorBoard callback'''\n",
    "    # define callback for early stopping\n",
    "    callback_list = []\n",
    "    es = EarlyStopping(monitor=es_monitor, mode=es_mode, verbose=1,patience = patience_threshold)\n",
    "    callback_list.append(es)\n",
    "    model_path = get_model_path()\n",
    "    save_model_path = os.path.join(model_path,'scmodel'+modifier+\"_\"+str(experiment_number)+'.h5')\n",
    "    # define callback to save best model\n",
    "    mc = ModelCheckpoint(save_model_path, monitor=es_monitor, mode=es_mode, verbose=1, save_best_only=True)\n",
    "    callback_list.append(mc)\n",
    "    # define callback for TensorBoard\n",
    "    if tensorboard_callback:\n",
    "        tensorboard_log_dir = os.path.join(get_path(),\"tensorboard_log\",datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\n",
    "        tensorboard = TensorBoard(log_dir= tensorboard_log_dir)\n",
    "        callback_list.append(tensorboard)\n",
    "    return(callback_list,save_model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text cols []\n",
      "dropout  0.0003\n",
      "L2 lambda  0.0003\n",
      "batch size  1000\n",
      "epochs 10\n",
      "learning_rate 0.001\n",
      "loss function binary_crossentropy\n",
      "output activation function hard_sigmoid\n",
      "patience_threshold is  15\n",
      "experiment number is  1\n",
      "early stop is False\n",
      "about to define embeddings\n",
      "textmax is 50\n",
      "through loops for cols\n",
      "through definition of non-text parts of main_l\n",
      "main_l Tensor(\"concatenate_17/Identity:0\", shape=(None, 64), dtype=float32)\n",
      "Train on 31292 samples, validate on 7824 samples\n",
      "Epoch 1/10\n",
      "31292/31292 [==============================] - 6s 179us/sample - loss: 4.5317 - accuracy: 0.4752 - accuracy_1: 0.4752 - val_loss: 1.8042 - val_accuracy: 0.6057 - val_accuracy_1: 0.6057\n",
      "Epoch 2/10\n",
      "31292/31292 [==============================] - 1s 22us/sample - loss: 1.3715 - accuracy: 0.7576 - accuracy_1: 0.7576 - val_loss: 1.1928 - val_accuracy: 0.6926 - val_accuracy_1: 0.6926\n",
      "Epoch 3/10\n",
      "31292/31292 [==============================] - 1s 28us/sample - loss: 0.7898 - accuracy: 0.8613 - accuracy_1: 0.8613 - val_loss: 1.1495 - val_accuracy: 0.7133 - val_accuracy_1: 0.7133\n",
      "Epoch 4/10\n",
      "31292/31292 [==============================] - 1s 21us/sample - loss: 0.6970 - accuracy: 0.9027 - accuracy_1: 0.9027 - val_loss: 1.0762 - val_accuracy: 0.7123 - val_accuracy_1: 0.7123\n",
      "Epoch 5/10\n",
      "31292/31292 [==============================] - 1s 26us/sample - loss: 0.6644 - accuracy: 0.9146 - accuracy_1: 0.9146 - val_loss: 1.1107 - val_accuracy: 0.7179 - val_accuracy_1: 0.7179\n",
      "Epoch 6/10\n",
      "31292/31292 [==============================] - 1s 21us/sample - loss: 0.6481 - accuracy: 0.9213 - accuracy_1: 0.9213 - val_loss: 1.0399 - val_accuracy: 0.7411 - val_accuracy_1: 0.7411\n",
      "Epoch 7/10\n",
      "31292/31292 [==============================] - 1s 23us/sample - loss: 0.6322 - accuracy: 0.9292 - accuracy_1: 0.9292 - val_loss: 1.0193 - val_accuracy: 0.7729 - val_accuracy_1: 0.7729\n",
      "Epoch 8/10\n",
      "31292/31292 [==============================] - 1s 23us/sample - loss: 0.6223 - accuracy: 0.9339 - accuracy_1: 0.9339 - val_loss: 0.9959 - val_accuracy: 0.8099 - val_accuracy_1: 0.8099\n",
      "Epoch 9/10\n",
      "31292/31292 [==============================] - 1s 22us/sample - loss: 0.6139 - accuracy: 0.9372 - accuracy_1: 0.9372 - val_loss: 0.9290 - val_accuracy: 0.8498 - val_accuracy_1: 0.8498\n",
      "Epoch 10/10\n",
      "31292/31292 [==============================] - 1s 24us/sample - loss: 0.6052 - accuracy: 0.9411 - accuracy_1: 0.9411 - val_loss: 0.8823 - val_accuracy: 0.8818 - val_accuracy_1: 0.8818\n",
      "Wall time: 27.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "print(\"text cols\",textcols)\n",
    "print(\"dropout \",dropout_rate)\n",
    "print(\"L2 lambda \",l2_lambda)\n",
    "print(\"batch size \",batch_size)\n",
    "print(\"epochs\",epochs)\n",
    "print(\"learning_rate\",learning_rate)\n",
    "print(\"loss function\",loss_func)\n",
    "print(\"output activation function\",output_activation)\n",
    "print(\"patience_threshold is \",patience_threshold)\n",
    "print(\"experiment number is \",experiment_number)\n",
    "print(\"early stop is\",early_stop)\n",
    "# get definitions\n",
    "callback_list, save_model_path = set_early_stop(es_monitor, es_mode)\n",
    "model = get_model()\n",
    "if early_stop:\n",
    "       modelfit = model.fit(X_train_list, dtrain.target, epochs=epochs, batch_size=batch_size\n",
    "        , validation_data=(X_valid_list, dvalid.target), class_weight = {0 : zero_weight, 1: one_weight}, verbose=1,callbacks=callback_list)\n",
    "else:\n",
    "    modelfit = model.fit(X_train_list, dtrain.target, epochs=epochs, batch_size=batch_size\n",
    "         , validation_data=(X_valid_list, dvalid.target), class_weight = {0 : zero_weight, 1: one_weight}, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved model, weights to disk\n"
     ]
    }
   ],
   "source": [
    "# save model elements explicitly if not saved as part of early_stop\n",
    "if early_stop == False:\n",
    "    model_json = model.to_json()\n",
    "    model_path = get_model_path()\n",
    "    with open(os.path.join(model_path,'model'+modifier+'.json'), \"w\") as json_file:\n",
    "        json_file.write(model_json)\n",
    "    # serialize weights to HDF5\n",
    "    model.save_weights(os.path.join(model_path,'scweights'+modifier+'.h5'))\n",
    "    save_model_path = os.path.join(model_path,'scmodel'+modifier+'.h5')\n",
    "    model.save(save_model_path,save_format='h5')\n",
    "    # no early stop, so make current model saved_model\n",
    "    saved_model = model\n",
    "    print(\"Saved model, weights to disk\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metrics names  ['loss', 'accuracy', 'accuracy_1']\n",
      "Train: 0.877, Test: 0.881\n"
     ]
    }
   ],
   "source": [
    "saved_model = load_model(save_model_path)\n",
    "print(\"metrics names \",saved_model.metrics_names)\n",
    "# getting test and train accuracy is time consuming - check to see if experiment requires it\n",
    "if get_test_train_acc:\n",
    "    if len(saved_model.metrics_names) == 2:\n",
    "        # saved_model.evaluate returns ['loss', 'acc']\n",
    "        _, train_acc = saved_model.evaluate(X_train_list, dtrain.target, verbose=0)\n",
    "        _, test_acc = saved_model.evaluate(X_test, test.target, verbose=0)\n",
    "    else:\n",
    "        # saved_model.evaluate returns ['loss', 'accuracy', 'accuracy_1']\n",
    "        _, train_acc,_ = saved_model.evaluate(X_train_list, dtrain.target, verbose=0)\n",
    "        _, test_acc,_ = saved_model.evaluate(X_test, test.target, verbose=0)\n",
    "    print('Train: %.3f, Test: %.3f' % (train_acc, test_acc))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([0, 0, 0, ..., 0, 1, 0], dtype=int64),\n",
       " array([1, 1, 1, ..., 1, 0, 1]),\n",
       " array([ 50,  94,  28, ..., 201, 214, 117]),\n",
       " array([306,  54, 281, ...,  33,   0,   0], dtype=int64),\n",
       " array([34460, 36850, 28945, ...,  9488,  3416, 33503], dtype=int64),\n",
       " array([3, 2, 1, ..., 2, 1, 3]),\n",
       " array([2., 1., 2., ..., 7., 3., 2.]),\n",
       " array([ 5.,  2., 12., ...,  1., 25., 21.]),\n",
       " array([1.15, 1.71, 0.44, ..., 0.11, 0.29, 3.25]),\n",
       " array([6., 3., 2., ..., 2., 1., 2.])]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 0, 1, 0], dtype=int64)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_list[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load saved model and weights <a name='reload' />\n",
    "<a href=#linkanchor>Back to link list</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "target (InputLayer)             [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "room_type (InputLayer)          [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_6 (Embedding)         (None, 1, 10)        20          target[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "embedding_7 (Embedding)         (None, 1, 10)        30          room_type[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "neighbourhood (InputLayer)      [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNor (None, 1, 10)        40          embedding_6[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNor (None, 1, 10)        40          embedding_7[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_8 (Embedding)         (None, 1, 10)        2210        neighbourhood[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "availability_365 (InputLayer)   [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_6 (Flatten)             (None, 10)           0           batch_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "flatten_7 (Flatten)             (None, 10)           0           batch_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNor (None, 1, 10)        40          embedding_8[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_9 (Embedding)         (None, 1, 10)        3660        availability_365[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "host_id (InputLayer)            [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_6 (Dropout)             (None, 10)           0           flatten_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_7 (Dropout)             (None, 10)           0           flatten_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_8 (Flatten)             (None, 10)           0           batch_normalization_8[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_9 (BatchNor (None, 1, 10)        40          embedding_9[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "embedding_10 (Embedding)        (None, 1, 10)        374570      host_id[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "neighbourhood_group (InputLayer [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_9 (Concatenate)     (None, 20)           0           dropout_6[0][0]                  \n",
      "                                                                 dropout_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_8 (Dropout)             (None, 10)           0           flatten_8[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_9 (Flatten)             (None, 10)           0           batch_normalization_9[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_10 (BatchNo (None, 1, 10)        40          embedding_10[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_11 (Embedding)        (None, 1, 10)        50          neighbourhood_group[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_10 (Concatenate)    (None, 30)           0           concatenate_9[0][0]              \n",
      "                                                                 dropout_8[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_9 (Dropout)             (None, 10)           0           flatten_9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "flatten_10 (Flatten)            (None, 10)           0           batch_normalization_10[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_11 (BatchNo (None, 1, 10)        40          embedding_11[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_11 (Concatenate)    (None, 40)           0           concatenate_10[0][0]             \n",
      "                                                                 dropout_9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_10 (Dropout)            (None, 10)           0           flatten_10[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_11 (Flatten)            (None, 10)           0           batch_normalization_11[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_12 (Concatenate)    (None, 50)           0           concatenate_11[0][0]             \n",
      "                                                                 dropout_10[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_11 (Dropout)            (None, 10)           0           flatten_11[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_13 (Concatenate)    (None, 60)           0           concatenate_12[0][0]             \n",
      "                                                                 dropout_11[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "minimum_nights (InputLayer)     [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_14 (Concatenate)    (None, 61)           0           concatenate_13[0][0]             \n",
      "                                                                 minimum_nights[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "number_of_reviews (InputLayer)  [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_15 (Concatenate)    (None, 62)           0           concatenate_14[0][0]             \n",
      "                                                                 number_of_reviews[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "reviews_per_month (InputLayer)  [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_16 (Concatenate)    (None, 63)           0           concatenate_15[0][0]             \n",
      "                                                                 reviews_per_month[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "calculated_host_listings_count  [(None, 1)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_17 (Concatenate)    (None, 64)           0           concatenate_16[0][0]             \n",
      "                                                                 calculated_host_listings_count[0]\n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 1)            65          concatenate_17[0][0]             \n",
      "==================================================================================================\n",
      "Total params: 380,845\n",
      "Trainable params: 380,725\n",
      "Non-trainable params: 120\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "saved_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "if presaved == True:\n",
    "    batch_size = 1000\n",
    "    epochs = 1\n",
    "    modelfit2 = loaded_model.fit(X_train_list, dtrain.target, epochs=epochs, batch_size=batch_size\n",
    "         , validation_data=(X_valid_list, dvalid.target), verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predictions and renderings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# functions to parse and manipulate dates in the style of the input CSVs\n",
    "\n",
    "def create_date(year,month):\n",
    "    outdate = datetime.date(year,month,15)\n",
    "    return(outdate)\n",
    "\n",
    "def parse_bic_date(bic_date_in):\n",
    "    year = int(bic_date_in[0:4])\n",
    "    month = int(bic_date_in[-2:])\n",
    "    return(year,month)\n",
    "\n",
    "def create_date_from_bic(bic_date_in):\n",
    "    yr,mth = parse_bic_date(bic_date_in)\n",
    "    retdate = create_date(yr,mth)\n",
    "    return retdate\n",
    "\n",
    "def get_datecomp_from_csvdate (csv_date):\n",
    "    day = int(csv_date[0:2])\n",
    "    # month_number = datetime.datetime.strptime(month_name, '%b').month\n",
    "    month = datetime.datetime.strptime(csv_date[3:6], '%b').month\n",
    "    year = int('20'+csv_date[-2:])\n",
    "    # year = int(csv_date[-2:])\n",
    "    return (year,month,day)\n",
    "\n",
    "def get_date_from_csvdate (csv_date):\n",
    "    day = int(csv_date[0:2])\n",
    "    # month_number = datetime.datetime.strptime(month_name, '%b').month\n",
    "    month = datetime.datetime.strptime(csv_date[3:6], '%b',coerce=True).month\n",
    "    year = int('20'+csv_date[-2:])\n",
    "    # year = int(csv_date[-2:])\n",
    "    return (date(year,month,day))\n",
    "\n",
    "def get_year_from_csvdate (csv_date):\n",
    "    year = int('20'+csv_date[-2:])\n",
    "    return (year)\n",
    "\n",
    "def get_month_from_csvdate (csv_date):\n",
    "    month = datetime.datetime.strptime(csv_date[3:6], '%b',coerce=True).month\n",
    "    return (month)\n",
    "\n",
    "def get_day_from_csvdate (csv_date):\n",
    "    day = int(csv_date[0:2])\n",
    "    return (day)\n",
    "\n",
    "def get_weekday (date):\n",
    "    return(date.weekday())\n",
    "\n",
    "# pd.to_datetime(x, coerce=True)\n",
    "\n",
    "def validatedate(csv_text):\n",
    "    try:\n",
    "        datetime.datetime.strptime(csv_date[3:6], '%b')\n",
    "    except ValueError:\n",
    "        raise ValueError(\"Incorrect data format, should be YYYY-MM-DD\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " ...]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(X_test_list[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_test  {'target': array([0, 0, 1, ..., 1, 0, 0], dtype=int64), 'room_type': array([1, 0, 0, ..., 0, 0, 1]), 'neighbourhood': array([194,  64,  85, ..., 209,  60,  94]), 'availability_365': array([351, 191,  67, ..., 168, 343,   0], dtype=int64), 'host_id': array([29704,  2172,  2641, ..., 22251, 17052,  4074], dtype=int64), 'neighbourhood_group': array([4, 2, 1, ..., 2, 1, 2]), 'minimum_nights': array([ 3.,  1., 30., ...,  3.,  3.,  2.]), 'number_of_reviews': array([26.,  5., 30., ...,  9., 62.,  7.]), 'reviews_per_month': array([1.11, 0.65, 0.39, ..., 0.2 , 1.19, 0.08]), 'calculated_host_listings_count': array([3., 1., 1., ..., 1., 1., 1.])}\n"
     ]
    }
   ],
   "source": [
    "print(\"X_test \", X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# predictions on training set\n",
    "\n",
    "preds = saved_model.predict(X_test, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9779, 1)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\personal\\manning\\tf2_test\\notebooks\\try_tf2\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAARc0lEQVR4nO3df5DcdX3H8edbUp1IBgiNnGmCDW3T1kDUMSelWjuXwSkR6gSmOg1lBCqdtAx26kxsDf1DnTqZ5h9sBxVtKg4w/rjJVBFGiC0TuTJWKCYOEgJSU8nEECYZhEaOMtTDd//YL7Ice7m9vd3v7t7n+ZjZud3Pfn+89rt3r/ved3e/F5mJJKkMr+p3AElSfSx9SSqIpS9JBbH0Jakglr4kFWRRvwPMZtmyZblq1aqO5n322Wc5+eSTuxuoB4YlJwxPVnN237BkHZac0Nuse/fufTIzX/eKOzJzoC/r1q3LTt19990dz1unYcmZOTxZzdl9w5J1WHJm9jYrsCdbdKqHdySpIJa+JBXE0pekglj6klQQS1+SCmLpS1JBLH1JKoilL0kFsfQlqSADfxoGaTartt7xirEta6e4ssV4Nx3cflFPly/1gnv6klQQS1+SCmLpS1JBLH1JKoilL0kFsfQlqSCWviQVxNKXpIJY+pJUEEtfkgpi6UtSQSx9SSqIpS9JBbH0Jakglr4kFcTSl6SCWPqSVBBLX5IKMmvpR8SZEXF3RDwSEfsj4q+q8dMj4q6I+GH1dWnTPNdGxIGIeDQiLmgaXxcR+6r7ro+I6M3DkiS10s6e/hSwJTPfCJwHXBMRa4CtwO7MXA3srm5T3bcJOBvYANwQESdVy/ossBlYXV02dPGxSJJmMWvpZ+YTmfm96vozwCPACmAjcHM12c3AxdX1jcB4Zj6fmY8BB4BzI2I5cEpm3puZCdzSNI8kqQbR6N82J45YBdwDnAMcyszTmu57OjOXRsSngfsy84vV+I3ALuAgsD0z31WNvxP4SGb+YYv1bKbxFwEjIyPrxsfHO3pwk5OTLFmypKN56zQsOWEws+57/PgrxkYWw9HnervetStOnfcyBnF7zmRYsg5LTuht1vXr1+/NzNHp44vaXUBELAG+CnwoM396gsPxre7IE4y/cjBzB7ADYHR0NMfGxtqN+TITExN0Om+dhiUnDGbWK7fe8YqxLWunuG5f29/eHTl42di8lzGI23Mmw5J1WHJCf7K29e6diPglGoX/pcz8WjV8tDpkQ/X1WDV+GDizafaVwJFqfGWLcUlSTdp5904ANwKPZOYnm+66Hbiiun4FcFvT+KaIeE1EnEXjBdv7M/MJ4JmIOK9a5uVN80iSatDO37/vAN4P7IuIB6qxvwW2Azsj4irgEPA+gMzcHxE7gYdpvPPnmsx8oZrvauAmYDGN4/y7uvQ4JEltmLX0M/PbtD4eD3D+DPNsA7a1GN9D40VgSVIf+IlcSSqIpS9JBbH0Jakglr4kFcTSl6SCWPqSVBBLX5IKYulLUkEsfUkqSG9PQ6hirGpxpktJg8c9fUkqiKUvSQWx9CWpIJa+JBXE0pekglj6klQQS1+SCmLpS1JBLH1JKoilL0kFsfQlqSCWviQVxNKXpIJY+pJUEEtfkgpi6UtSQSx9SSqIpS9JBbH0Jakglr4kFcTSl6SCWPqSVBBLX5IKYulLUkEsfUkqiKUvSQWx9CWpILOWfkR8ISKORcRDTWMfj4jHI+KB6nJh033XRsSBiHg0Ii5oGl8XEfuq+66PiOj+w5EknUg7e/o3ARtajP9DZr6lutwJEBFrgE3A2dU8N0TESdX0nwU2A6urS6tlSpJ6aNbSz8x7gKfaXN5GYDwzn8/Mx4ADwLkRsRw4JTPvzcwEbgEu7jS0JKkz8zmm/8GIeLA6/LO0GlsB/LhpmsPV2Irq+vRxSVKNorHjPctEEauAb2TmOdXtEeBJIIFPAMsz8wMR8Rng3sz8YjXdjcCdwCHg7zPzXdX4O4G/ycz3zLC+zTQOBTEyMrJufHy8owc3OTnJkiVLOpq3TsOSE2bOuu/x431IM7ORxXD0ud6uY+2KU+e9jIXw3A+aYckJvc26fv36vZk5On18UScLy8yjL16PiH8GvlHdPAyc2TTpSuBINb6yxfhMy98B7AAYHR3NsbGxTmIyMTFBp/PWaVhywsxZr9x6R/1hTmDL2imu29fRt3fbDl42Nu9lLITnftAMS07oT9aODu9Ux+hfdAnw4jt7bgc2RcRrIuIsGi/Y3p+ZTwDPRMR51bt2Lgdum0duSVIHZt0VioivAGPAsog4DHwMGIuIt9A4vHMQ+HOAzNwfETuBh4Ep4JrMfKFa1NU03gm0GNhVXSRJNZq19DPz0hbDN55g+m3Athbje4Bz5pROktRVfiJXkgpi6UtSQSx9SSqIpS9JBbH0Jakglr4kFcTSl6SCWPqSVBBLX5IK0tszUvXZvseP9+VEYAe3X1T7OiWpHe7pS1JBLH1JKoilL0kFsfQlqSCWviQVxNKXpIJY+pJUEEtfkgpi6UtSQSx9SSqIpS9JBbH0Jakglr4kFcTSl6SCWPqSVBBLX5IKYulLUkEsfUkqiKUvSQWx9CWpIJa+JBXE0pekglj6klQQS1+SCmLpS1JBLH1JKoilL0kFmbX0I+ILEXEsIh5qGjs9Iu6KiB9WX5c23XdtRByIiEcj4oKm8XURsa+67/qIiO4/HEnSibSzp38TsGHa2FZgd2auBnZXt4mINcAm4Oxqnhsi4qRqns8Cm4HV1WX6MiVJPTZr6WfmPcBT04Y3AjdX128GLm4aH8/M5zPzMeAAcG5ELAdOycx7MzOBW5rmkSTVpNNj+iOZ+QRA9fWManwF8OOm6Q5XYyuq69PHJUk1WtTl5bU6Tp8nGG+9kIjNNA4FMTIywsTEREdhRhbDlrVTHc07H3PNOzk52fFjrNtMWfuxnU+kjue+G8/ZQnjuB82w5IT+ZO209I9GxPLMfKI6dHOsGj8MnNk03UrgSDW+ssV4S5m5A9gBMDo6mmNjYx2F/NSXbuO6fd3+vTa7g5eNzWn6iYkJOn2MdZsp65Vb76g/zAlsWTvV8+d+rs9zK8P23F/5zWf7su6D2y9qe9ph26Z1Z+308M7twBXV9SuA25rGN0XEayLiLBov2N5fHQJ6JiLOq961c3nTPJKkmsy6KxQRXwHGgGURcRj4GLAd2BkRVwGHgPcBZOb+iNgJPAxMAddk5gvVoq6m8U6gxcCu6iJJqtGspZ+Zl85w1/kzTL8N2NZifA9wzpzSSZK6yk/kSlJBLH1JKoilL0kFsfQlqSCWviQVxNKXpIJY+pJUEEtfkgpi6UtSQeo/G5m0QKzqwknmtqyd6uhkdXM5AZnUzD19SSqIpS9JBfHwzgLTjUMOJ9Lp4QhJg8E9fUkqiKUvSQWx9CWpIJa+JBXE0pekglj6klQQS1+SCmLpS1JB/HCWpIE3lw8ddvsDhAvtPEfu6UtSQSx9SSqIpS9JBbH0Jakglr4kFcTSl6SCWPqSVBBLX5IKYulLUkEsfUkqiKUvSQWx9CWpIJa+JBXE0pekglj6klQQS1+SCjKv0o+IgxGxLyIeiIg91djpEXFXRPyw+rq0afprI+JARDwaERfMN7wkaW66sae/PjPfkpmj1e2twO7MXA3srm4TEWuATcDZwAbghog4qQvrlyS1qReHdzYCN1fXbwYubhofz8znM/Mx4ABwbg/WL0maQWRm5zNHPAY8DSTwT5m5IyL+JzNPa5rm6cxcGhGfBu7LzC9W4zcCuzLzX1osdzOwGWBkZGTd+Ph4R/mOPXWco891NOu8rF1x6pymn5ycZMmSJV1Z977Hj3dlOTMZWUxftulcLfScc/0e64bJyUkeO/5C7eudq24/973c1t382Z9u/fr1e5uOwPzCfP8x+jsy80hEnAHcFRE/OMG00WKs5W+czNwB7AAYHR3NsbGxjsJ96ku3cd2++v/3+8HLxuY0/cTEBJ0+xum6+Q+hW9mydqov23SuFnrOuX6PdcPExATXffvZ2tc7V91+7nu5rbv5s9+ueW2ZzDxSfT0WEbfSOFxzNCKWZ+YTEbEcOFZNfhg4s2n2lcCR+ax/UK2aY/FuWTvV87KWJJhH6UfEycCrMvOZ6vofAH8H3A5cAWyvvt5WzXI78OWI+CTwK8Bq4P55ZJeKNdcdi27YsnaK+R8cUL/N5xkcAW6NiBeX8+XM/GZEfBfYGRFXAYeA9wFk5v6I2Ak8DEwB12Tm4B8glKQFpOPSz8wfAW9uMf4T4PwZ5tkGbOt0nZKk+fETuZJUEEtfkgpi6UtSQSx9SSqIpS9JBbH0Jakglr4kFcTSl6SCWPqSVBBLX5IKYulLUkEsfUkqiKUvSQWx9CWpIJa+JBXE0pekglj6klQQS1+SCmLpS1JBLH1JKoilL0kFsfQlqSCWviQVxNKXpIJY+pJUEEtfkgpi6UtSQSx9SSqIpS9JBbH0Jakglr4kFcTSl6SCWPqSVBBLX5IKsqjfASRpkK3aekfPlr1l7RRXzrD8g9sv6sk63dOXpIJY+pJUkNpLPyI2RMSjEXEgIrbWvX5JKlmtpR8RJwGfAd4NrAEujYg1dWaQpJLVvad/LnAgM3+Umf8HjAMba84gScWKzKxvZRHvBTZk5p9Vt98P/E5mfnDadJuBzdXN3wIe7XCVy4AnO5y3TsOSE4Ynqzm7b1iyDktO6G3WX83M100frPstm9Fi7BW/dTJzB7Bj3iuL2JOZo/NdTq8NS04Ynqzm7L5hyTosOaE/Wes+vHMYOLPp9krgSM0ZJKlYdZf+d4HVEXFWRLwa2ATcXnMGSSpWrYd3MnMqIj4I/CtwEvCFzNzfw1XO+xBRTYYlJwxPVnN237BkHZac0Iestb6QK0nqLz+RK0kFsfQlqSALovRnO7VDNFxf3f9gRLx1QHP+dkTcGxHPR8SH+5GxyjFbzsuq7fhgRHwnIt7cj5xVltmybqxyPhAReyLi9wYxZ9N0b4uIF6rPtPRFG9t0LCKOV9v0gYj46CDmrKYZqzLuj4h/rztjlWG27fnXTdvyoer5P71ngTJzqC80XhD+b+DXgFcD3wfWTJvmQmAXjc8JnAf854DmPAN4G7AN+PAAb8+3A0ur6+/ux/acQ9YlvPTa1ZuAHwxizqbpvgXcCbx3gLfpGPCNfuSbY87TgIeBN1S3zxjEnNOmfw/wrV5mWgh7+u2c2mEjcEs23AecFhHLBy1nZh7LzO8CP6s5W7N2cn4nM5+ubt5H4/MW/dBO1smsfpqAk2nxYcAatHv6kb8EvgocqzPcNMNyqpR2cv4J8LXMPASNn6+aM8Lct+elwFd6GWghlP4K4MdNtw9XY3OdptcGIUM75przKhp/RfVDW1kj4pKI+AFwB/CBmrI1mzVnRKwALgE+V2OuVtp9/n83Ir4fEbsi4ux6or1MOzl/E1gaERMRsTciLq8t3Uva/nmKiNcCG2j84u+ZhfCfs9o5tUNbp3/osUHI0I62c0bEehql35fj5LR/Wo9bgVsj4veBTwDv6nWwadrJ+Y/ARzLzhYhWk9emnazfo3Fel8mIuBD4OrC658lerp2ci4B1wPnAYuDeiLgvM/+r1+GazOXn/j3Af2TmUz3MsyBKv51TOwzC6R8GIUM72soZEW8CPg+8OzN/UlO26ea0TTPznoj49YhYlpl1npCrnZyjwHhV+MuACyNiKjO/Xk/EX5g1a2b+tOn6nRFxw4Bu08PAk5n5LPBsRNwDvBmos/Tn8j26iR4f2gEWxAu5i4AfAWfx0gslZ0+b5iJe/kLu/YOYs2naj9O/F3Lb2Z5vAA4Abx+C5/43eOmF3LcCj794e5ByTpv+Jvr3Qm472/T1Tdv0XODQIG5T4I3A7mra1wIPAecMWs5qulOBp4CTe51p6Pf0c4ZTO0TEX1T3f47GuyEupFFU/wv86SDmjIjXA3uAU4CfR8SHaLzS/9MZF9yHnMBHgV8Gbqj2TKeyD2c1bDPrHwGXR8TPgOeAP87qp2zAcg6ENrO+F7g6IqZobNNNg7hNM/ORiPgm8CDwc+DzmfnQoOWsJr0E+Lds/FXSU56GQZIKshDevSNJapOlL0kFsfQlqSCWviQVxNKXpIJY+pJUEEtfkgry/2xpmPjX+2+IAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "test[\"predict\"] = preds\n",
    "test.predict[:5]\n",
    "if verboseout:\n",
    "    test.predict.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD7CAYAAACG50QgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAWFklEQVR4nO3cf5DUd33H8edLMBGjNGBkPSEWrPiDgFE5KWrHOYstmFSJrZlemgradK5laEZnsA30D9tOh5n8k46SMbFMtMA0lWHUCKNBRczWsZIg0RhCEpozQXJCQ02qCbGDOXz3j/2k+eZY7r537H3vvvm8HjM7+93Pfj7ffe1y99rlu3uriMDMzPLwookOYGZm1XHpm5llxKVvZpYRl76ZWUZc+mZmGXHpm5llZMTSl/QGSfcUTk9K+rikmZL2SHoonc8orNkgqV/SYUnLC+OLJR1M122SpPG6Y2ZmdiaN5nP6kqYAPwV+G1gLPBER10taD8yIiOskLQC+ACwBXg18C3h9RJyWtB/4GHAncDuwKSJ2d/QemZnZWU0d5fxlwI8j4ieSVgI9aXwr0ASuA1YC2yPiFPCIpH5giaQjwPSI2AcgaRtwBTBs6V900UUxd+7cUcZsefrpp7ngggvGtLZKdckJ9cnqnJ1Xl6x1yQnjm/Xuu+/+WUS8cuj4aEu/l9areIBGRBwHiIjjkmal8dm0Xsk/ayCNPZO2h44Pa+7cuRw4cGCUMVuazSY9PT1jWluluuSE+mR1zs6rS9a65ITxzSrpJ+3GS5e+pPOADwAbRpraZiyGGW93W31AH0Cj0aDZbJaN+TwnT54c89oq1SUn1Cerc3ZeXbLWJSdMUNaIKHWiddjmm4XLh4GutN0FHE7bG4ANhXnfAN6R5jxYGL8K+OeRbnfx4sUxVnfccceY11apLjkj6pPVOTuvLlnrkjNifLMCB6JNp47mI5tX8dyhHYBdwOq0vRrYWRjvlXS+pHnAfGB/tA4FPSVpafrUzqrCGjMzq0CpwzuSXgr8HvAXheHrgR2SrgGOAlcCRMQhSTuA+4FBYG1EnE5r1gBbgGm03sD1J3fMzCpUqvQj4pfAK4aMPU7r0zzt5m8ENrYZPwAsHH1MMzPrBP9FrplZRlz6ZmYZcembmWXEpW9mlpHR/kVurRz86S/4yPqvTXSMEa1bNFiLnFCfrFXkPHL95eO6f7Px4Ff6ZmYZcembmWXEpW9mlhGXvplZRlz6ZmYZcembmWXEpW9mlhGXvplZRlz6ZmYZcembmWXEpW9mlhGXvplZRlz6ZmYZcembmWXEpW9mlhGXvplZRlz6ZmYZKVX6ki6U9EVJD0p6QNI7JM2UtEfSQ+l8RmH+Bkn9kg5LWl4YXyzpYLpukySNx50yM7P2yr7S/zTw9Yh4I3Ap8ACwHtgbEfOBvekykhYAvcAlwArgJklT0n5uBvqA+em0okP3w8zMShix9CVNB94NfA4gIn4VET8HVgJb07StwBVpeyWwPSJORcQjQD+wRFIXMD0i9kVEANsKa8zMrAJlXum/Fvhv4F8k/VDSLZIuABoRcRwgnc9K82cDjxbWD6Sx2Wl76LiZmVVkask5bwOujYi7JH2adCjnLNodp49hxs/cgdRH6zAQjUaDZrNZIuaZGtNg3aLBMa2tUl1yQn2yVpFzrD+XRSdPnuzIfqpQl6x1yQkTk7VM6Q8AAxFxV7r8RVql/5ikrog4ng7dnCjMv7iwfg5wLI3PaTN+hojYDGwG6O7ujp6ennL3Zogbb93JDQfL3MWJtW7RYC1yQn2yVpHzyNU957yPZrPJWH++q1aXrHXJCROTdcTDOxHxX8Cjkt6QhpYB9wO7gNVpbDWwM23vAnolnS9pHq03bPenQ0BPSVqaPrWzqrDGzMwqUPal0LXArZLOAx4GPkrrCWOHpGuAo8CVABFxSNIOWk8Mg8DaiDid9rMG2AJMA3ank5mZVaRU6UfEPUB3m6uWnWX+RmBjm/EDwMLRBDQzs87xX+SamWXEpW9mlhGXvplZRlz6ZmYZcembmWXEpW9mlhGXvplZRlz6ZmYZcembmWXEpW9mlhGXvplZRlz6ZmYZcembmWXEpW9mlhGXvplZRlz6ZmYZcembmWXEpW9mlhGXvplZRlz6ZmYZcembmWXEpW9mlpFSpS/piKSDku6RdCCNzZS0R9JD6XxGYf4GSf2SDktaXhhfnPbTL2mTJHX+LpmZ2dmM5pX+eyLiLRHRnS6vB/ZGxHxgb7qMpAVAL3AJsAK4SdKUtOZmoA+Yn04rzv0umJlZWedyeGclsDVtbwWuKIxvj4hTEfEI0A8skdQFTI+IfRERwLbCGjMzq0DZ0g/gm5LultSXxhoRcRwgnc9K47OBRwtrB9LY7LQ9dNzMzCoyteS8d0XEMUmzgD2SHhxmbrvj9DHM+Jk7aD2x9AE0Gg2azWbJmM/XmAbrFg2OaW2V6pIT6pO1ipxj/bksOnnyZEf2U4W6ZK1LTpiYrKVKPyKOpfMTkm4DlgCPSeqKiOPp0M2JNH0AuLiwfA5wLI3PaTPe7vY2A5sBuru7o6enp/QdKrrx1p3ccLDs89rEWbdosBY5oT5Zq8h55Oqec95Hs9lkrD/fVatL1rrkhInJOuLhHUkXSHr5s9vA7wP3AbuA1WnaamBn2t4F9Eo6X9I8Wm/Y7k+HgJ6StDR9amdVYY2ZmVWgzEuhBnBb+nTlVODfIuLrkr4P7JB0DXAUuBIgIg5J2gHcDwwCayPidNrXGmALMA3YnU5mZlaREUs/Ih4GLm0z/jiw7CxrNgIb24wfABaOPqaZmXWC/yLXzCwjLn0zs4y49M3MMuLSNzPLiEvfzCwjLn0zs4y49M3MMuLSNzPLiEvfzCwjLn0zs4y49M3MMuLSNzPLiEvfzCwjLn0zs4y49M3MMuLSNzPLiEvfzCwjLn0zs4y49M3MMuLSNzPLiEvfzCwjLn0zs4yULn1JUyT9UNJX0+WZkvZIeiidzyjM3SCpX9JhScsL44slHUzXbZKkzt4dMzMbzmhe6X8MeKBweT2wNyLmA3vTZSQtAHqBS4AVwE2SpqQ1NwN9wPx0WnFO6c3MbFRKlb6kOcDlwC2F4ZXA1rS9FbiiML49Ik5FxCNAP7BEUhcwPSL2RUQA2wprzMysAlNLzvsU8DfAywtjjYg4DhARxyXNSuOzgTsL8wbS2DNpe+j4GST10fofAY1Gg2azWTLm8zWmwbpFg2NaW6W65IT6ZK0i51h/LotOnjzZkf1UoS5Z65ITJibriKUv6Q+AExFxt6SeEvtsd5w+hhk/czBiM7AZoLu7O3p6ytzsmW68dSc3HCz7vDZx1i0arEVOqE/WKnIeubrnnPfRbDYZ68931eqStS45YWKylvmteBfwAUmXAS8Bpkv6V+AxSV3pVX4XcCLNHwAuLqyfAxxL43PajJuZWUVGPKYfERsiYk5EzKX1Bu23I+JPgV3A6jRtNbAzbe8CeiWdL2kerTds96dDQU9JWpo+tbOqsMbMzCpwLv//vR7YIeka4ChwJUBEHJK0A7gfGATWRsTptGYNsAWYBuxOJzMzq8ioSj8imkAzbT8OLDvLvI3AxjbjB4CFow1pZmad4b/INTPLiEvfzCwjLn0zs4y49M3MMuLSNzPLiEvfzCwjLn0zs4y49M3MMuLSNzPLiEvfzCwjLn0zs4y49M3MMuLSNzPLiEvfzCwjLn0zs4y49M3MMuLSNzPLiEvfzCwjLn0zs4y49M3MMuLSNzPLiEvfzCwjI5a+pJdI2i/pR5IOSfqHND5T0h5JD6XzGYU1GyT1SzosaXlhfLGkg+m6TZI0PnfLzMzamVpizingdyPipKQXA9+VtBv4Q2BvRFwvaT2wHrhO0gKgF7gEeDXwLUmvj4jTwM1AH3AncDuwAtjd8XtlVoG56792zvtYt2iQj3RgP1WoS9a65IThsx65/vJxuc0RX+lHy8l08cXpFMBKYGsa3wpckbZXAtsj4lREPAL0A0skdQHTI2JfRASwrbDGzMwqUOaVPpKmAHcDrwM+ExF3SWpExHGAiDguaVaaPpvWK/lnDaSxZ9L20PF2t9dH638ENBoNms1m6TtU1JjWeiad7OqSE+qT1Tk7ry5Z65IThs861t4bSanST4dm3iLpQuA2SQuHmd7uOH0MM97u9jYDmwG6u7ujp6enTMwz3HjrTm44WOouTqh1iwZrkRPqk9U5O68uWeuSE4bPeuTqnnG5zVF9eicifg40aR2LfywdsiGdn0jTBoCLC8vmAMfS+Jw242ZmVpEyn955ZXqFj6RpwHuBB4FdwOo0bTWwM23vAnolnS9pHjAf2J8OBT0laWn61M6qwhozM6tAmf8DdQFb03H9FwE7IuKrkvYBOyRdAxwFrgSIiEOSdgD3A4PA2nR4CGANsAWYRutTO/7kjplZhUYs/Yi4F3hrm/HHgWVnWbMR2Nhm/AAw3PsBZmY2jvwXuWZmGXHpm5llxKVvZpYRl76ZWUZc+mZmGXHpm5llxKVvZpYRl76ZWUZc+mZmGXHpm5llxKVvZpYRl76ZWUZc+mZmGXHpm5llxKVvZpYRl76ZWUZc+mZmGXHpm5llxKVvZpYRl76ZWUZc+mZmGRmx9CVdLOkOSQ9IOiTpY2l8pqQ9kh5K5zMKazZI6pd0WNLywvhiSQfTdZskaXzulpmZtVPmlf4gsC4i3gQsBdZKWgCsB/ZGxHxgb7pMuq4XuARYAdwkaUra181AHzA/nVZ08L6YmdkIRiz9iDgeET9I208BDwCzgZXA1jRtK3BF2l4JbI+IUxHxCNAPLJHUBUyPiH0REcC2whozM6vAqI7pS5oLvBW4C2hExHFoPTEAs9K02cCjhWUDaWx22h46bmZmFZladqKklwFfAj4eEU8Oczi+3RUxzHi72+qjdRiIRqNBs9ksG/N5GtNg3aLBMa2tUl1yQn2yOmfn1SVrXXLC8FnH2nsjKVX6kl5Mq/BvjYgvp+HHJHVFxPF06OZEGh8ALi4snwMcS+Nz2oyfISI2A5sBuru7o6enp9y9GeLGW3dyw8HSz2sTZt2iwVrkhPpkdc7Oq0vWuuSE4bMeubpnXG6zzKd3BHwOeCAi/qlw1S5gddpeDewsjPdKOl/SPFpv2O5Ph4CekrQ07XNVYY2ZmVWgzNPhu4APAwcl3ZPG/ha4Htgh6RrgKHAlQEQckrQDuJ/WJ3/WRsTptG4NsAWYBuxOJzMzq8iIpR8R36X98XiAZWdZsxHY2Gb8ALBwNAHNzKxz/Be5ZmYZcembmWXEpW9mlhGXvplZRlz6ZmYZcembmWXEpW9mlhGXvplZRlz6ZmYZcembmWXEpW9mlhGXvplZRlz6ZmYZcembmWXEpW9mlhGXvplZRlz6ZmYZcembmWXEpW9mlhGXvplZRlz6ZmYZcembmWVkxNKX9HlJJyTdVxibKWmPpIfS+YzCdRsk9Us6LGl5YXyxpIPpuk2S1Pm7Y2ZmwynzSn8LsGLI2Hpgb0TMB/amy0haAPQCl6Q1N0maktbcDPQB89Np6D7NzGycjVj6EfEd4IkhwyuBrWl7K3BFYXx7RJyKiEeAfmCJpC5gekTsi4gAthXWmJlZRcZ6TL8REccB0vmsND4beLQwbyCNzU7bQ8fNzKxCUzu8v3bH6WOY8fY7kfpoHQqi0WjQbDbHFKYxDdYtGhzT2irVJSfUJ6tzdl5dstYlJwyfday9N5Kxlv5jkroi4ng6dHMijQ8AFxfmzQGOpfE5bcbbiojNwGaA7u7u6OnpGVPIG2/dyQ0HO/281nnrFg3WIifUJ6tzdl5dstYlJwyf9cjVPeNym2M9vLMLWJ22VwM7C+O9ks6XNI/WG7b70yGgpyQtTZ/aWVVYY2ZmFRnx6VDSF4Ae4CJJA8DfAdcDOyRdAxwFrgSIiEOSdgD3A4PA2og4nXa1htYngaYBu9PJzMwqNGLpR8RVZ7lq2VnmbwQ2thk/ACwcVTozM+so/0WumVlGXPpmZhlx6ZuZZcSlb2aWEZe+mVlGXPpmZhlx6ZuZZcSlb2aWEZe+mVlGXPpmZhlx6ZuZZcSlb2aWEZe+mVlGXPpmZhlx6ZuZZcSlb2aWEZe+mVlGXPpmZhlx6ZuZZcSlb2aWEZe+mVlGXPpmZhmpvPQlrZB0WFK/pPVV376ZWc4qLX1JU4DPAO8DFgBXSVpQZQYzs5xV/Up/CdAfEQ9HxK+A7cDKijOYmWWr6tKfDTxauDyQxszMrAKKiOpuTLoSWB4Rf54ufxhYEhHXDpnXB/Sli28ADo/xJi8CfjbGtVWqS06oT1bn7Ly6ZK1LThjfrL8ZEa8cOjh1nG7sbAaAiwuX5wDHhk6KiM3A5nO9MUkHIqL7XPcz3uqSE+qT1Tk7ry5Z65ITJiZr1Yd3vg/MlzRP0nlAL7Cr4gxmZtmq9JV+RAxK+ivgG8AU4PMRcajKDGZmOav68A4RcTtwe0U3d86HiCpSl5xQn6zO2Xl1yVqXnDABWSt9I9fMzCaWv4bBzCwjL4jSH+mrHdSyKV1/r6S3TdKcb5S0T9IpSZ+YiIwpx0g5r06P472Svifp0onImbKMlHVlynmPpAOSfmcy5izMe7uk05I+VGW+IRlGekx7JP0iPab3SPrkZMyZ5vSkjIck/XvVGVOGkR7Pvy48lvelf/+Z4xYoImp9ovWG8I+B1wLnAT8CFgyZcxmwGxCwFLhrkuacBbwd2Ah8YhI/nu8EZqTt903E4zmKrC/jucOYbwYenIw5C/O+Tes9rw9N4se0B/jqROQbZc4LgfuB16TLsyZjziHz3w98ezwzvRBe6Zf5aoeVwLZouRO4UFLXZMsZESci4vvAMxVnKyqT83sR8T/p4p20/t5iIpTJejLSbxNwATARb2KV/fqRa4EvASeqDDdEXb4qpUzOPwG+HBFHofX7VXFGGP3jeRXwhfEM9EIo/TJf7TAZvv5hMmQoY7Q5r6H1v6iJUCqrpA9KehD4GvBnFWUrGjGnpNnAB4HPVpirnbL//u+Q9CNJuyVdUk205ymT8/XADElNSXdLWlVZuueU/n2S9FJgBa0n/nFT+Uc2x4HajA19NVdmznibDBnKKJ1T0ntolf6EHCenZNaIuA24TdK7gX8E3jvewYYok/NTwHURcVpqN70yZbL+gNaf+J+UdBnwFWD+uCd7vjI5pwKLgWXANGCfpDsj4j/HO1zBaH7v3w/8R0Q8MY55XhClX+arHUp9/cM4mwwZyiiVU9KbgVuA90XE4xVlG2pUj2lEfEfSb0m6KCKq/G6WMjm7ge2p8C8CLpM0GBFfqSbi/xsxa0Q8Wdi+XdJNk/QxHQB+FhFPA09L+g5wKVBl6Y/mZ7SXcT60A7wg3sidCjwMzOO5N0ouGTLncp7/Ru7+yZizMPfvmbg3css8nq8B+oF31uDf/nU890bu24CfPnt5MuUcMn8LE/dGbpnH9FWFx3QJcHQyPqbAm4C9ae5LgfuAhZMtZ5r3G8ATwAXjnan2r/TjLF/tIOkv0/WfpfVpiMtoFdUvgY9OxpySXgUcAKYDv5b0cVrv9D951h1PQE7gk8ArgJvSK9PBmIAvuCqZ9Y+AVZKeAf4X+ONIv2WTLOekUDLrh4A1kgZpPaa9k/ExjYgHJH0duBf4NXBLRNw32XKmqR8Evhmt/5WMK/9FrplZRl4In94xM7OSXPpmZhlx6ZuZZcSlb2aWEZe+mVlGXPpmZhlx6ZuZZcSlb2aWkf8DcwCHol6pUHIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "if verboseout:\n",
    "    test.predict.hist(bins=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\personal\\manning\\tf2_test\\notebooks\\try_tf2\\lib\\site-packages\\ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "25022    0\n",
       "38145    0\n",
       "2095     0\n",
       "16738    0\n",
       "26601    0\n",
       "Name: predround, dtype: int32"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get rounded predictions\n",
    "test[\"predround\"] = preds.round().astype(int)\n",
    "test.predround[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test target 0 6830\n",
      "test target 1 2949\n",
      "test predround 0 7995\n",
      "test predround 1 1784\n"
     ]
    }
   ],
   "source": [
    "print(\"test target 0\",test[test['target']==0].shape[0])\n",
    "print(\"test target 1\",test[test['target']==1].shape[0])\n",
    "print(\"test predround 0\",test[test['predround']==0].shape[0])\n",
    "print(\"test predround 1\",test[test['predround']==1].shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1165\n",
      "percentage correct test\n",
      "0.8808671643317313\n"
     ]
    }
   ],
   "source": [
    "# get delta between predictions on training set and actual training target values\n",
    "# hand calculate accuracy on training set as ratio of (total training samples - wrong training predictions)/total training samples\n",
    "\n",
    "deltatr = abs(test.target[:100000] - test.predround[:100000])\n",
    "deltatr[:50]\n",
    "print(deltatr.sum())\n",
    "print(\"percentage correct test\")\n",
    "print((len(deltatr) - deltatr.sum())/len(deltatr))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18225    0.110300\n",
       "30496    0.178325\n",
       "24126    0.267897\n",
       "8313     0.222473\n",
       "4594     0.218024\n",
       "Name: predict, dtype: float32"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# predict values for validation X values\n",
    "# X_valid, dvalid.target\n",
    "predval = model.predict(X_valid, batch_size=batch_size)\n",
    "dvalid[\"predround\"] = predval.round().astype(int)\n",
    "dvalid[\"predict\"] = predval\n",
    "#print(type(deltaval))\n",
    "#print(len(deltaval))\n",
    "dvalid.predict[:5]\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAASiUlEQVR4nO3cf6zd9X3f8eeruKUujGBGuEU2Hax1twJu0vqWsWaZrgsqDklrogXVGStux2QV0amV3A7YH92myRL/MFWQQmolFUZhtSy11FZSt0NO7qK1MGJ3JMYQhhss6oKwEgjBLKK55L0/zjfNibn2PfeHv/fA5/mQjs45n/P5nPP6Hp/78jnf8yNVhSSpDd+33AEkSf2x9CWpIZa+JDXE0pekhlj6ktSQFcsdYC4XXnhhXXrppQta+/rrr3POOecsbaAlZsbFG/d8YMalYsbRHTx48KtV9e63XFBVY31Yv359LdTnPve5Ba/tixkXb9zzVZlxqZhxdMCBmqVT3b0jSQ2x9CWpIZa+JDXE0pekhlj6ktQQS1+SGmLpS1JDLH1JaoilL0kNGfufYViMQ3/7Kr9yx2eWO8ZpbVs3Y8ZFWq58R+/6YO+3KS2Wz/QlqSGWviQ1xNKXpIZY+pLUEEtfkhpi6UtSQyx9SWqIpS9JDbH0Jakhlr4kNcTSl6SGWPqS1BBLX5IaYulLUkMsfUlqiKUvSQ2x9CWpIZa+JDXE0pekhoxU+kmOJjmU5IkkB7qxC5I8kuTZ7njV0Pw7kxxJ8kyS64bG13fXcyTJPUmy9JskSTqV+TzT31BV762qye78HcD+qloL7O/Ok+RyYDNwBbARuC/JWd2a+4GtwNrusHHxmyBJGtVidu9sAnZ2p3cCNwyN76qqN6rqOeAIcFWSi4HzqurRqirgwaE1kqQeZNC/c0xKngNeAQr4/arakeTrVXX+0JxXqmpVko8Bj1XVp7rxTwL7gKPAXVV1bTf+fuD2qvrQLLe3lcErAiYmJtbv2rVrQRt3/OVXeembC1ram4mVmHGRlivfutXvGnnuiRMnOPfcc89gmsUz49IYl4wbNmw4OLRn5u+tGHH9+6rqhSQXAY8k+fJp5s62n75OM/7WwaodwA6AycnJmpqaGjHm97r3oT3cfWjUTVwe29bNmHGRlivf0ZumRp47PT3NQh/HfTHj0hj3jCPt3qmqF7rj48DDwFXAS90uG7rj4930Y8AlQ8vXAC9042tmGZck9WTO0k9yTpJ/8J3TwM8DTwJ7gS3dtC3Anu70XmBzkrOTXMbgDdvHq+pF4LUkV3ef2rl5aI0kqQejvCaeAB7uPl25AvjvVfVnSb4A7E5yC/A8cCNAVR1Osht4CpgBbquqN7vruhV4AFjJYD//viXcFknSHOYs/ar6CvCeWca/BlxzijXbge2zjB8Arpx/TEnSUvAbuZLUEEtfkhpi6UtSQyx9SWqIpS9JDbH0Jakhlr4kNcTSl6SGWPqS1BBLX5IaYulLUkMsfUlqiKUvSQ2x9CWpIZa+JDXE0pekhlj6ktQQS1+SGmLpS1JDLH1JaoilL0kNsfQlqSGWviQ1xNKXpIZY+pLUEEtfkhpi6UtSQ0Yu/SRnJfk/ST7dnb8gySNJnu2OVw3NvTPJkSTPJLluaHx9kkPdZfckydJujiTpdObzTP83gKeHzt8B7K+qtcD+7jxJLgc2A1cAG4H7kpzVrbkf2Aqs7Q4bF5VekjQvI5V+kjXAB4FPDA1vAnZ2p3cCNwyN76qqN6rqOeAIcFWSi4HzqurRqirgwaE1kqQejPpM/3eB/wB8e2hsoqpeBOiOL+rGVwN/MzTvWDe2ujt98rgkqScr5pqQ5EPA8ao6mGRqhOucbT99nWZ8ttvcymA3EBMTE0xPT49ws281sRK2rZtZ0Nq+mHHxlivffB6XJ06cWPDjuC9mXBrjnnHO0gfeB/xikuuBHwTOS/Ip4KUkF1fVi92um+Pd/GPAJUPr1wAvdONrZhl/i6raAewAmJycrKmpqdG3aMi9D+3h7kOjbOLy2bZuxoyLtFz5jt40NfLc6elpFvo47osZl8a4Z5xz905V3VlVa6rqUgZv0H62qv4NsBfY0k3bAuzpTu8FNic5O8llDN6wfbzbBfRakqu7T+3cPLRGktSDxTw9ugvYneQW4HngRoCqOpxkN/AUMAPcVlVvdmtuBR4AVgL7uoMkqSfzKv2qmgamu9NfA645xbztwPZZxg8AV843pCRpafiNXElqiKUvSQ2x9CWpIZa+JDXE0pekhlj6ktQQS1+SGmLpS1JDLH1JaoilL0kNsfQlqSGWviQ1xNKXpIZY+pLUEEtfkhpi6UtSQyx9SWqIpS9JDbH0Jakhlr4kNcTSl6SGWPqS1BBLX5IaYulLUkMsfUlqiKUvSQ2x9CWpIZa+JDVkztJP8oNJHk/yxSSHk/yXbvyCJI8kebY7XjW05s4kR5I8k+S6ofH1SQ51l92TJGdmsyRJsxnlmf4bwM9V1XuA9wIbk1wN3AHsr6q1wP7uPEkuBzYDVwAbgfuSnNVd1/3AVmBtd9i4hNsiSZrDnKVfAye6s9/fHQrYBOzsxncCN3SnNwG7quqNqnoOOAJcleRi4LyqerSqCnhwaI0kqQcZ9O8ckwbP1A8CPwb8XlXdnuTrVXX+0JxXqmpVko8Bj1XVp7rxTwL7gKPAXVV1bTf+fuD2qvrQLLe3lcErAiYmJtbv2rVrQRt3/OVXeembC1ram4mVmHGRlivfutXvGnnuiRMnOPfcc89gmsUz49IYl4wbNmw4WFWTJ4+vGGVxVb0JvDfJ+cDDSa48zfTZ9tPXacZnu70dwA6AycnJmpqaGiXmW9z70B7uPjTSJi6bbetmzLhIy5Xv6E1TI8+dnp5moY/jvphxaYx7xnl9eqeqvg5MM9gX/1K3y4bu+Hg37RhwydCyNcAL3fiaWcYlST0Z5dM77+6e4ZNkJXAt8GVgL7Clm7YF2NOd3gtsTnJ2kssYvGH7eFW9CLyW5OruUzs3D62RJPVglNfEFwM7u/363wfsrqpPJ3kU2J3kFuB54EaAqjqcZDfwFDAD3NbtHgK4FXgAWMlgP/++pdwYSdLpzVn6VfUl4KdmGf8acM0p1mwHts8yfgA43fsBkqQzyG/kSlJDLH1JaoilL0kNsfQlqSGWviQ1xNKXpIZY+pLUEEtfkhpi6UtSQyx9SWqIpS9JDbH0Jakhlr4kNcTSl6SGWPqS1BBLX5IaYulLUkMsfUlqiKUvSQ2x9CWpIZa+JDXE0pekhlj6ktQQS1+SGrJiuQNIb1eX3vGZkeduWzfDr8xj/nIw49JYqoxH7/rgEqR5K5/pS1JDLH1JaoilL0kNmbP0k1yS5HNJnk5yOMlvdOMXJHkkybPd8aqhNXcmOZLkmSTXDY2vT3Kou+yeJDkzmyVJms0oz/RngG1V9RPA1cBtSS4H7gD2V9VaYH93nu6yzcAVwEbgviRnddd1P7AVWNsdNi7htkiS5jBn6VfVi1X1V93p14CngdXAJmBnN20ncEN3ehOwq6reqKrngCPAVUkuBs6rqkerqoAHh9ZIknqQQf+OODm5FPg8cCXwfFWdP3TZK1W1KsnHgMeq6lPd+CeBfcBR4K6qurYbfz9we1V9aJbb2crgFQETExPrd+3ataCNO/7yq7z0zQUt7c3ESsy4SOOeD8y4VFrKuG71uxa1fsOGDQeravLk8ZE/p5/kXOCPgN+sqm+cZnf8bBfUacbfOli1A9gBMDk5WVNTU6PG/B73PrSHuw+N91cRtq2bMeMijXs+MONSaSnj0ZumFh9mFiN9eifJ9zMo/Ieq6o+74Ze6XTZ0x8e78WPAJUPL1wAvdONrZhmXJPVklE/vBPgk8HRV/behi/YCW7rTW4A9Q+Obk5yd5DIGb9g+XlUvAq8lubq7zpuH1kiSejDKa5D3Ab8MHEryRDf2H4G7gN1JbgGeB24EqKrDSXYDTzH45M9tVfVmt+5W4AFgJYP9/PuWaDskSSOYs/Sr6n8x+/54gGtOsWY7sH2W8QMM3gSWJC0Dv5ErSQ2x9CWpIZa+JDXE0pekhlj6ktQQS1+SGmLpS1JDLH1JaoilL0kNsfQlqSGWviQ1xNKXpIZY+pLUEEtfkhpi6UtSQyx9SWqIpS9JDbH0Jakhlr4kNcTSl6SGWPqS1BBLX5IaYulLUkMsfUlqiKUvSQ2x9CWpIZa+JDXE0pekhsxZ+kn+IMnxJE8OjV2Q5JEkz3bHq4YuuzPJkSTPJLluaHx9kkPdZfckydJvjiTpdEZ5pv8AsPGksTuA/VW1FtjfnSfJ5cBm4IpuzX1JzurW3A9sBdZ2h5OvU5J0hs1Z+lX1eeDlk4Y3ATu70zuBG4bGd1XVG1X1HHAEuCrJxcB5VfVoVRXw4NAaSVJPVixw3URVvQhQVS8muagbXw08NjTvWDf2re70yeOzSrKVwasCJiYmmJ6eXljIlbBt3cyC1vbFjIs37vnAjEulpYwL7b25LLT0T2W2/fR1mvFZVdUOYAfA5ORkTU1NLSjMvQ/t4e5DS72JS2vbuhkzLtK45wMzLpWWMh69aWrxYWax0E/vvNTtsqE7Pt6NHwMuGZq3BnihG18zy7gkqUcLLf29wJbu9BZgz9D45iRnJ7mMwRu2j3e7gl5LcnX3qZ2bh9ZIknoy52uQJH8ITAEXJjkG/CfgLmB3kluA54EbAarqcJLdwFPADHBbVb3ZXdWtDD4JtBLY1x0kST2as/Sr6qOnuOiaU8zfDmyfZfwAcOW80kmSlpTfyJWkhlj6ktQQS1+SGmLpS1JDLH1JaoilL0kNsfQlqSGWviQ1xNKXpIZY+pLUEEtfkhpi6UtSQyx9SWqIpS9JDbH0Jakhlr4kNcTSl6SGWPqS1BBLX5IaYulLUkMsfUlqiKUvSQ2x9CWpIZa+JDXE0pekhlj6ktQQS1+SGmLpS1JDei/9JBuTPJPkSJI7+r59SWpZr6Wf5Czg94APAJcDH01yeZ8ZJKllfT/Tvwo4UlVfqaq/A3YBm3rOIEnNSlX1d2PJR4CNVfXvuvO/DPyzqvr1k+ZtBbZ2Z/8J8MwCb/JC4KsLXNsXMy7euOcDMy4VM47uH1XVu08eXNFziMwy9pb/dapqB7Bj0TeWHKiqycVez5lkxsUb93xgxqVixsXre/fOMeCSofNrgBd6ziBJzeq79L8ArE1yWZIfADYDe3vOIEnN6nX3TlXNJPl14M+Bs4A/qKrDZ/AmF72LqAdmXLxxzwdmXCpmXKRe38iVJC0vv5ErSQ2x9CWpIe+I0p/rpx0ycE93+ZeS/PSY5funSR5N8kaS3+oz2zwy3tTdd19K8pdJ3jOGGTd1+Z5IciDJvxi3jEPzfibJm913V3o1wv04leTV7n58IsnvjFvGoZxPJDmc5H+OU74kvz10/z3Z/Vtf0GfGU6qqt/WBwRvCfw38Y+AHgC8Cl58053pgH4PvCVwN/O8xy3cR8DPAduC3xvQ+/FlgVXf6A33eh/PIeC7ffZ/qJ4Evj1vGoXmfBf4U+Mi4ZQSmgE/3/TicZ8bzgaeAH+nOXzRO+U6a/wvAZ5fr/jz58E54pj/KTztsAh6sgceA85NcPC75qup4VX0B+FZPmU42Ssa/rKpXurOPMfiOxbhlPFHdXxlwDrN88W+5M3b+PfBHwPE+w3XeDj+FMkrGfw38cVU9D4O/oTHLN+yjwB/2kmwE74TSXw38zdD5Y93YfOecKct526Oab8ZbGLxy6tNIGZN8OMmXgc8A/7anbN8xZ8Ykq4EPAx/vMdewUf+t/3mSLybZl+SKfqL9vVEy/jiwKsl0koNJbu4t3Tz+XpL8ELCRwX/yY6Hvn2E4E0b5aYeRfv7hDFnO2x7VyBmTbGBQ+n3vLx/1JzweBh5O8i+B/wpce6aDDRkl4+8Ct1fVm8ls08+4UTL+FYPfbTmR5HrgT4C1ZzzZd42ScQWwHrgGWAk8muSxqvq/Zzoc8/ub/gXgL6rq5TOYZ17eCaU/yk87LOfPP7wdfnpipIxJfhL4BPCBqvpaT9m+Y173Y1V9PsmPJrmwqvr68atRMk4Cu7rCvxC4PslMVf1JPxHnzlhV3xg6/adJ7hvD+/EY8NWqeh14PcnngfcAfZT+fB6LmxmjXTvAO+KN3BXAV4DL+O6bKlecNOeDfO8buY+PU76huf+Z5Xkjd5T78EeAI8DPjvG/84/x3Tdyfxr42++cH5eMJ81/gP7fyB3lfvzhofvxKuD5cbsfgZ8A9ndzfwh4ErhyXPJ1894FvAyc0+e/8VyHt/0z/TrFTzsk+bXu8o8z+JTE9QxK6/8BvzpO+ZL8MHAAOA/4dpLfZPBpgG+c8op7zgj8DvAPgfu6Z6kz1eMvCY6Y8V8BNyf5FvBN4Jeq++sbo4zLasSMHwFuTTLD4H7cPG73Y1U9neTPgC8B3wY+UVVPjku+buqHgf9Rg1cjY8OfYZCkhrwTPr0jSRqRpS9JDbH0Jakhlr4kNcTSl6SGWPqS1BBLX5Ia8v8B/JxEuFrPYbsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "if verboseout:\n",
    "    dvalid.predict.hist(bins=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18225    0\n",
      "30496    0\n",
      "24126    0\n",
      "8313     0\n",
      "4594     0\n",
      "153      0\n",
      "17698    0\n",
      "35735    1\n",
      "42207    0\n",
      "30760    0\n",
      "Name: deltaval, dtype: int64\n",
      "925\n"
     ]
    }
   ],
   "source": [
    "# hand calculation of proportion correct guesses in validation set\n",
    "\n",
    "dvalid[\"deltaval\"] = abs(dvalid.target - dvalid.predround)\n",
    "print(dvalid[\"deltaval\"][:10])\n",
    "print(dvalid[\"deltaval\"].sum())\n",
    "# print(\"percentage correct\")\n",
    "# print((len(deltaval) - deltaval.sum())/len(deltaval))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25022    0\n",
      "38145    0\n",
      "2095     1\n",
      "16738    0\n",
      "26601    0\n",
      "42449    0\n",
      "42566    0\n",
      "2814     0\n",
      "41521    0\n",
      "15027    0\n",
      "Name: deltaval, dtype: int64\n",
      "1165\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\personal\\manning\\tf2_test\\notebooks\\try_tf2\\lib\\site-packages\\ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "# hand calculation of proportion correct guesses in validation set\n",
    "\n",
    "test[\"deltaval\"] = abs(test.target - test.predround)\n",
    "print(test[\"deltaval\"][:10])\n",
    "print(test[\"deltaval\"].sum())\n",
    "# print(\"percentage correct\")\n",
    "# print((len(deltaval) - deltaval.sum())/len(deltaval))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "      <th>host_id</th>\n",
       "      <th>host_name</th>\n",
       "      <th>neighbourhood_group</th>\n",
       "      <th>neighbourhood</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>room_type</th>\n",
       "      <th>price</th>\n",
       "      <th>minimum_nights</th>\n",
       "      <th>number_of_reviews</th>\n",
       "      <th>last_review</th>\n",
       "      <th>reviews_per_month</th>\n",
       "      <th>calculated_host_listings_count</th>\n",
       "      <th>availability_365</th>\n",
       "      <th>target</th>\n",
       "      <th>predround</th>\n",
       "      <th>predict</th>\n",
       "      <th>deltaval</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>35735</th>\n",
       "      <td>28359765</td>\n",
       "      <td>STUDIO WITH OUTDOOR SPACE-CHELSEA MARKET-WEST ...</td>\n",
       "      <td>33868</td>\n",
       "      <td>Pranjal</td>\n",
       "      <td>2</td>\n",
       "      <td>34</td>\n",
       "      <td>40.74138</td>\n",
       "      <td>-73.99729</td>\n",
       "      <td>0</td>\n",
       "      <td>175.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-06-13</td>\n",
       "      <td>1.00</td>\n",
       "      <td>65.0</td>\n",
       "      <td>342</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15574</th>\n",
       "      <td>12542544</td>\n",
       "      <td>Near Times Square</td>\n",
       "      <td>26016</td>\n",
       "      <td>Michael</td>\n",
       "      <td>2</td>\n",
       "      <td>95</td>\n",
       "      <td>40.76231</td>\n",
       "      <td>-73.99520</td>\n",
       "      <td>0</td>\n",
       "      <td>225.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>2019-07-07</td>\n",
       "      <td>3.39</td>\n",
       "      <td>1.0</td>\n",
       "      <td>218</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.380657</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48510</th>\n",
       "      <td>36304117</td>\n",
       "      <td>Gorgeous 4 BR Apt in the Heart of Nolita!</td>\n",
       "      <td>37392</td>\n",
       "      <td>Margaret</td>\n",
       "      <td>2</td>\n",
       "      <td>145</td>\n",
       "      <td>40.72349</td>\n",
       "      <td>-73.99457</td>\n",
       "      <td>0</td>\n",
       "      <td>316.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2019-01-01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>310</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.471676</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12256</th>\n",
       "      <td>9486229</td>\n",
       "      <td>Stylish and clean apt, sleeps 5</td>\n",
       "      <td>23865</td>\n",
       "      <td>Carine</td>\n",
       "      <td>2</td>\n",
       "      <td>92</td>\n",
       "      <td>40.72797</td>\n",
       "      <td>-74.00190</td>\n",
       "      <td>0</td>\n",
       "      <td>350.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>47.0</td>\n",
       "      <td>2019-05-15</td>\n",
       "      <td>1.06</td>\n",
       "      <td>2.0</td>\n",
       "      <td>283</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.460032</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11987</th>\n",
       "      <td>9336526</td>\n",
       "      <td>Small room with convenient commute</td>\n",
       "      <td>19046</td>\n",
       "      <td>Xiangyu</td>\n",
       "      <td>3</td>\n",
       "      <td>68</td>\n",
       "      <td>40.73682</td>\n",
       "      <td>-73.87500</td>\n",
       "      <td>1</td>\n",
       "      <td>180.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2019-01-01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.454761</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24766</th>\n",
       "      <td>19887942</td>\n",
       "      <td>Unique 3 bedroom apartment in Bed-Stuy come all</td>\n",
       "      <td>31267</td>\n",
       "      <td>Edwin</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>40.68286</td>\n",
       "      <td>-73.93379</td>\n",
       "      <td>0</td>\n",
       "      <td>170.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>75.0</td>\n",
       "      <td>2019-07-02</td>\n",
       "      <td>3.24</td>\n",
       "      <td>1.0</td>\n",
       "      <td>323</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.466620</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4004</th>\n",
       "      <td>2514689</td>\n",
       "      <td>3 BEDS IN LOVELY 2BEDROOM/2BATH MIDTOWN RENTAL</td>\n",
       "      <td>13088</td>\n",
       "      <td>Jada</td>\n",
       "      <td>2</td>\n",
       "      <td>127</td>\n",
       "      <td>40.75879</td>\n",
       "      <td>-73.96413</td>\n",
       "      <td>0</td>\n",
       "      <td>399.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>87.0</td>\n",
       "      <td>2017-12-18</td>\n",
       "      <td>1.37</td>\n",
       "      <td>3.0</td>\n",
       "      <td>267</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.369364</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33045</th>\n",
       "      <td>26063230</td>\n",
       "      <td>Gorgeous 4 BR apt w. King bed 30 min to Manhattan</td>\n",
       "      <td>2882</td>\n",
       "      <td>Sarah</td>\n",
       "      <td>1</td>\n",
       "      <td>74</td>\n",
       "      <td>40.64240</td>\n",
       "      <td>-73.95365</td>\n",
       "      <td>0</td>\n",
       "      <td>165.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>2019-07-02</td>\n",
       "      <td>4.04</td>\n",
       "      <td>6.0</td>\n",
       "      <td>191</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.313338</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7416</th>\n",
       "      <td>5510725</td>\n",
       "      <td>Fun in the Heart of Manhattan</td>\n",
       "      <td>9785</td>\n",
       "      <td>Mekado</td>\n",
       "      <td>2</td>\n",
       "      <td>95</td>\n",
       "      <td>40.76181</td>\n",
       "      <td>-73.98757</td>\n",
       "      <td>0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>2019-06-09</td>\n",
       "      <td>1.06</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.472380</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11788</th>\n",
       "      <td>9170129</td>\n",
       "      <td>New Apt -2 Blocks CENTRAL PARK Near COLUMBIA UNIV</td>\n",
       "      <td>23368</td>\n",
       "      <td>Lorena</td>\n",
       "      <td>2</td>\n",
       "      <td>94</td>\n",
       "      <td>40.79939</td>\n",
       "      <td>-73.95470</td>\n",
       "      <td>0</td>\n",
       "      <td>249.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>107.0</td>\n",
       "      <td>2019-06-23</td>\n",
       "      <td>2.40</td>\n",
       "      <td>2.0</td>\n",
       "      <td>249</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.343698</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24491</th>\n",
       "      <td>19702498</td>\n",
       "      <td>Sunny 2BR Brooklyn Loft</td>\n",
       "      <td>7921</td>\n",
       "      <td>Samuel</td>\n",
       "      <td>1</td>\n",
       "      <td>28</td>\n",
       "      <td>40.70766</td>\n",
       "      <td>-73.92169</td>\n",
       "      <td>0</td>\n",
       "      <td>175.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2018-09-09</td>\n",
       "      <td>0.34</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.487400</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13572</th>\n",
       "      <td>10154444</td>\n",
       "      <td>Cozy apartment (East Williamsburg)</td>\n",
       "      <td>2564</td>\n",
       "      <td>Helena</td>\n",
       "      <td>1</td>\n",
       "      <td>214</td>\n",
       "      <td>40.71275</td>\n",
       "      <td>-73.94299</td>\n",
       "      <td>0</td>\n",
       "      <td>175.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2019-05-26</td>\n",
       "      <td>0.05</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.465082</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31053</th>\n",
       "      <td>24097816</td>\n",
       "      <td>1BR Apartment on the Upper East Side.</td>\n",
       "      <td>4064</td>\n",
       "      <td>Daniel</td>\n",
       "      <td>2</td>\n",
       "      <td>201</td>\n",
       "      <td>40.78124</td>\n",
       "      <td>-73.94988</td>\n",
       "      <td>0</td>\n",
       "      <td>250.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2019-01-01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.357379</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22376</th>\n",
       "      <td>18074899</td>\n",
       "      <td>Stunning 2 bedroom Apt - Breathtaking views!!</td>\n",
       "      <td>19574</td>\n",
       "      <td>Kara</td>\n",
       "      <td>2</td>\n",
       "      <td>95</td>\n",
       "      <td>40.76191</td>\n",
       "      <td>-73.99766</td>\n",
       "      <td>0</td>\n",
       "      <td>499.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2019-01-01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>121.0</td>\n",
       "      <td>185</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1365</th>\n",
       "      <td>600286</td>\n",
       "      <td>lovely, spacious wmsbrg apt w deck</td>\n",
       "      <td>2300</td>\n",
       "      <td>George</td>\n",
       "      <td>1</td>\n",
       "      <td>214</td>\n",
       "      <td>40.70397</td>\n",
       "      <td>-73.95536</td>\n",
       "      <td>0</td>\n",
       "      <td>180.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>2019-05-13</td>\n",
       "      <td>0.36</td>\n",
       "      <td>1.0</td>\n",
       "      <td>312</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.473803</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26152</th>\n",
       "      <td>20857788</td>\n",
       "      <td>Two bedroom Beauty in midtown Manhattan</td>\n",
       "      <td>30110</td>\n",
       "      <td>Stanley</td>\n",
       "      <td>2</td>\n",
       "      <td>137</td>\n",
       "      <td>40.74844</td>\n",
       "      <td>-73.97694</td>\n",
       "      <td>0</td>\n",
       "      <td>367.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2018-08-19</td>\n",
       "      <td>0.09</td>\n",
       "      <td>50.0</td>\n",
       "      <td>358</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1342</th>\n",
       "      <td>588677</td>\n",
       "      <td>Great find- 2 bedroom apartment in Williamsburg!</td>\n",
       "      <td>4419</td>\n",
       "      <td>Michael</td>\n",
       "      <td>1</td>\n",
       "      <td>214</td>\n",
       "      <td>40.70989</td>\n",
       "      <td>-73.95347</td>\n",
       "      <td>0</td>\n",
       "      <td>250.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2012-08-10</td>\n",
       "      <td>0.01</td>\n",
       "      <td>2.0</td>\n",
       "      <td>250</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.418976</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11347</th>\n",
       "      <td>8814936</td>\n",
       "      <td>Lovely Home, Heart of Williamsburg!</td>\n",
       "      <td>6659</td>\n",
       "      <td>Paulina</td>\n",
       "      <td>1</td>\n",
       "      <td>214</td>\n",
       "      <td>40.71554</td>\n",
       "      <td>-73.96246</td>\n",
       "      <td>0</td>\n",
       "      <td>325.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>264.0</td>\n",
       "      <td>2019-06-16</td>\n",
       "      <td>5.85</td>\n",
       "      <td>2.0</td>\n",
       "      <td>47</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.046397</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11814</th>\n",
       "      <td>9194620</td>\n",
       "      <td>NYC Loft Living Close to Everything!</td>\n",
       "      <td>5380</td>\n",
       "      <td>Jonathan</td>\n",
       "      <td>2</td>\n",
       "      <td>201</td>\n",
       "      <td>40.76050</td>\n",
       "      <td>-73.96053</td>\n",
       "      <td>0</td>\n",
       "      <td>275.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2016-08-20</td>\n",
       "      <td>0.09</td>\n",
       "      <td>1.0</td>\n",
       "      <td>253</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.486353</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18283</th>\n",
       "      <td>14340165</td>\n",
       "      <td>Luxury Modern Studio in The Bronx</td>\n",
       "      <td>11127</td>\n",
       "      <td>Chondra</td>\n",
       "      <td>0</td>\n",
       "      <td>192</td>\n",
       "      <td>40.81587</td>\n",
       "      <td>-73.81532</td>\n",
       "      <td>0</td>\n",
       "      <td>175.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>2019-05-28</td>\n",
       "      <td>0.92</td>\n",
       "      <td>1.0</td>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.477804</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             id                                               name  host_id  \\\n",
       "35735  28359765  STUDIO WITH OUTDOOR SPACE-CHELSEA MARKET-WEST ...    33868   \n",
       "15574  12542544                                  Near Times Square    26016   \n",
       "48510  36304117          Gorgeous 4 BR Apt in the Heart of Nolita!    37392   \n",
       "12256   9486229                    Stylish and clean apt, sleeps 5    23865   \n",
       "11987   9336526                 Small room with convenient commute    19046   \n",
       "24766  19887942    Unique 3 bedroom apartment in Bed-Stuy come all    31267   \n",
       "4004    2514689     3 BEDS IN LOVELY 2BEDROOM/2BATH MIDTOWN RENTAL    13088   \n",
       "33045  26063230  Gorgeous 4 BR apt w. King bed 30 min to Manhattan     2882   \n",
       "7416    5510725                      Fun in the Heart of Manhattan     9785   \n",
       "11788   9170129  New Apt -2 Blocks CENTRAL PARK Near COLUMBIA UNIV    23368   \n",
       "24491  19702498                            Sunny 2BR Brooklyn Loft     7921   \n",
       "13572  10154444                 Cozy apartment (East Williamsburg)     2564   \n",
       "31053  24097816              1BR Apartment on the Upper East Side.     4064   \n",
       "22376  18074899      Stunning 2 bedroom Apt - Breathtaking views!!    19574   \n",
       "1365     600286                 lovely, spacious wmsbrg apt w deck     2300   \n",
       "26152  20857788            Two bedroom Beauty in midtown Manhattan    30110   \n",
       "1342     588677   Great find- 2 bedroom apartment in Williamsburg!     4419   \n",
       "11347   8814936                Lovely Home, Heart of Williamsburg!     6659   \n",
       "11814   9194620               NYC Loft Living Close to Everything!     5380   \n",
       "18283  14340165                  Luxury Modern Studio in The Bronx    11127   \n",
       "\n",
       "      host_name  neighbourhood_group  neighbourhood  latitude  longitude  \\\n",
       "35735   Pranjal                    2             34  40.74138  -73.99729   \n",
       "15574   Michael                    2             95  40.76231  -73.99520   \n",
       "48510  Margaret                    2            145  40.72349  -73.99457   \n",
       "12256    Carine                    2             92  40.72797  -74.00190   \n",
       "11987   Xiangyu                    3             68  40.73682  -73.87500   \n",
       "24766     Edwin                    1             13  40.68286  -73.93379   \n",
       "4004       Jada                    2            127  40.75879  -73.96413   \n",
       "33045     Sarah                    1             74  40.64240  -73.95365   \n",
       "7416     Mekado                    2             95  40.76181  -73.98757   \n",
       "11788    Lorena                    2             94  40.79939  -73.95470   \n",
       "24491    Samuel                    1             28  40.70766  -73.92169   \n",
       "13572    Helena                    1            214  40.71275  -73.94299   \n",
       "31053    Daniel                    2            201  40.78124  -73.94988   \n",
       "22376      Kara                    2             95  40.76191  -73.99766   \n",
       "1365     George                    1            214  40.70397  -73.95536   \n",
       "26152   Stanley                    2            137  40.74844  -73.97694   \n",
       "1342    Michael                    1            214  40.70989  -73.95347   \n",
       "11347   Paulina                    1            214  40.71554  -73.96246   \n",
       "11814  Jonathan                    2            201  40.76050  -73.96053   \n",
       "18283   Chondra                    0            192  40.81587  -73.81532   \n",
       "\n",
       "       room_type  price  minimum_nights  number_of_reviews last_review  \\\n",
       "35735          0  175.0            30.0                1.0  2019-06-13   \n",
       "15574          0  225.0             4.0              128.0  2019-07-07   \n",
       "48510          0  316.0            30.0                0.0  2019-01-01   \n",
       "12256          0  350.0             4.0               47.0  2019-05-15   \n",
       "11987          1  180.0             1.0                0.0  2019-01-01   \n",
       "24766          0  170.0             3.0               75.0  2019-07-02   \n",
       "4004           0  399.0             1.0               87.0  2017-12-18   \n",
       "33045          0  165.0             2.0               49.0  2019-07-02   \n",
       "7416           0  160.0             2.0               53.0  2019-06-09   \n",
       "11788          0  249.0             7.0              107.0  2019-06-23   \n",
       "24491          0  175.0             5.0                8.0  2018-09-09   \n",
       "13572          0  175.0             3.0                2.0  2019-05-26   \n",
       "31053          0  250.0            50.0                0.0  2019-01-01   \n",
       "22376          0  499.0            30.0                0.0  2019-01-01   \n",
       "1365           0  180.0            14.0               27.0  2019-05-13   \n",
       "26152          0  367.0            30.0                1.0  2018-08-19   \n",
       "1342           0  250.0            30.0                1.0  2012-08-10   \n",
       "11347          0  325.0             1.0              264.0  2019-06-16   \n",
       "11814          0  275.0            30.0                4.0  2016-08-20   \n",
       "18283          0  175.0             2.0               28.0  2019-05-28   \n",
       "\n",
       "       reviews_per_month  calculated_host_listings_count  availability_365  \\\n",
       "35735               1.00                            65.0               342   \n",
       "15574               3.39                             1.0               218   \n",
       "48510               0.00                             1.0               310   \n",
       "12256               1.06                             2.0               283   \n",
       "11987               0.00                             2.0                 0   \n",
       "24766               3.24                             1.0               323   \n",
       "4004                1.37                             3.0               267   \n",
       "33045               4.04                             6.0               191   \n",
       "7416                1.06                             1.0                 0   \n",
       "11788               2.40                             2.0               249   \n",
       "24491               0.34                             1.0                 0   \n",
       "13572               0.05                             3.0                 0   \n",
       "31053               0.00                             1.0                 0   \n",
       "22376               0.00                           121.0               185   \n",
       "1365                0.36                             1.0               312   \n",
       "26152               0.09                            50.0               358   \n",
       "1342                0.01                             2.0               250   \n",
       "11347               5.85                             2.0                47   \n",
       "11814               0.09                             1.0               253   \n",
       "18283               0.92                             1.0                36   \n",
       "\n",
       "       target  predround   predict  deltaval  \n",
       "35735       1          0  0.000000         1  \n",
       "15574       1          0  0.380657         1  \n",
       "48510       1          0  0.471676         1  \n",
       "12256       1          0  0.460032         1  \n",
       "11987       1          0  0.454761         1  \n",
       "24766       1          0  0.466620         1  \n",
       "4004        1          0  0.369364         1  \n",
       "33045       1          0  0.313338         1  \n",
       "7416        1          0  0.472380         1  \n",
       "11788       1          0  0.343698         1  \n",
       "24491       1          0  0.487400         1  \n",
       "13572       1          0  0.465082         1  \n",
       "31053       1          0  0.357379         1  \n",
       "22376       1          0  0.000000         1  \n",
       "1365        1          0  0.473803         1  \n",
       "26152       1          0  0.000000         1  \n",
       "1342        1          0  0.418976         1  \n",
       "11347       1          0  0.046397         1  \n",
       "11814       1          0  0.486353         1  \n",
       "18283       1          0  0.477804         1  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get subset of dataframe with wrong guesses\n",
    "# k1 = df.loc[(df.Product == p_id)\n",
    "dvalidwrong = dvalid.loc[(dvalid.deltaval == 1)]\n",
    "dvalidright = dvalid.loc[(dvalid.deltaval == 0)]\n",
    "dvalidwrong.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "      <th>host_id</th>\n",
       "      <th>host_name</th>\n",
       "      <th>neighbourhood_group</th>\n",
       "      <th>neighbourhood</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>room_type</th>\n",
       "      <th>price</th>\n",
       "      <th>minimum_nights</th>\n",
       "      <th>number_of_reviews</th>\n",
       "      <th>last_review</th>\n",
       "      <th>reviews_per_month</th>\n",
       "      <th>calculated_host_listings_count</th>\n",
       "      <th>availability_365</th>\n",
       "      <th>target</th>\n",
       "      <th>predict</th>\n",
       "      <th>predround</th>\n",
       "      <th>deltaval</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2095</th>\n",
       "      <td>949713</td>\n",
       "      <td>Luxury Duplex Loft w. Private Roof &amp; Amazing V...</td>\n",
       "      <td>2641</td>\n",
       "      <td>Tal</td>\n",
       "      <td>1</td>\n",
       "      <td>85</td>\n",
       "      <td>40.67450</td>\n",
       "      <td>-73.99582</td>\n",
       "      <td>0</td>\n",
       "      <td>400.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>2019-06-01</td>\n",
       "      <td>0.39</td>\n",
       "      <td>1.0</td>\n",
       "      <td>67</td>\n",
       "      <td>1</td>\n",
       "      <td>0.353480</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7752</th>\n",
       "      <td>5873714</td>\n",
       "      <td>East Village -  Beautiful 1 Bedroom</td>\n",
       "      <td>19637</td>\n",
       "      <td>Majeed</td>\n",
       "      <td>2</td>\n",
       "      <td>64</td>\n",
       "      <td>40.73182</td>\n",
       "      <td>-73.98450</td>\n",
       "      <td>1</td>\n",
       "      <td>195.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2016-11-28</td>\n",
       "      <td>0.37</td>\n",
       "      <td>1.0</td>\n",
       "      <td>178</td>\n",
       "      <td>1</td>\n",
       "      <td>0.485379</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45063</th>\n",
       "      <td>34511584</td>\n",
       "      <td>Fifth Avenue Ultra Luxurious Large 3 Bed- Gym/...</td>\n",
       "      <td>26073</td>\n",
       "      <td>John</td>\n",
       "      <td>2</td>\n",
       "      <td>127</td>\n",
       "      <td>40.76261</td>\n",
       "      <td>-73.97680</td>\n",
       "      <td>0</td>\n",
       "      <td>560.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2019-01-01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>337</td>\n",
       "      <td>1</td>\n",
       "      <td>0.495361</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43019</th>\n",
       "      <td>33400202</td>\n",
       "      <td>Four Points by Sheraton New York Downtown-7 Ni...</td>\n",
       "      <td>7694</td>\n",
       "      <td>Yun</td>\n",
       "      <td>2</td>\n",
       "      <td>73</td>\n",
       "      <td>40.70832</td>\n",
       "      <td>-74.00755</td>\n",
       "      <td>1</td>\n",
       "      <td>169.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-05-17</td>\n",
       "      <td>0.57</td>\n",
       "      <td>13.0</td>\n",
       "      <td>358</td>\n",
       "      <td>1</td>\n",
       "      <td>0.008758</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31743</th>\n",
       "      <td>24752747</td>\n",
       "      <td>Sonder | Hanover Square | Simple 1BR + Rooftop</td>\n",
       "      <td>12806</td>\n",
       "      <td>Sonder</td>\n",
       "      <td>2</td>\n",
       "      <td>73</td>\n",
       "      <td>40.70518</td>\n",
       "      <td>-74.00994</td>\n",
       "      <td>0</td>\n",
       "      <td>182.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2019-04-01</td>\n",
       "      <td>0.24</td>\n",
       "      <td>96.0</td>\n",
       "      <td>347</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089</th>\n",
       "      <td>457829</td>\n",
       "      <td>Beautiful apartment in the heart of The Village</td>\n",
       "      <td>4222</td>\n",
       "      <td>Lorena</td>\n",
       "      <td>2</td>\n",
       "      <td>64</td>\n",
       "      <td>40.72917</td>\n",
       "      <td>-73.98811</td>\n",
       "      <td>0</td>\n",
       "      <td>210.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>178.0</td>\n",
       "      <td>2019-07-01</td>\n",
       "      <td>2.06</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.239129</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33358</th>\n",
       "      <td>26357145</td>\n",
       "      <td>Nice one bedroom near United Nations</td>\n",
       "      <td>30110</td>\n",
       "      <td>Stanley</td>\n",
       "      <td>2</td>\n",
       "      <td>137</td>\n",
       "      <td>40.75030</td>\n",
       "      <td>-73.97605</td>\n",
       "      <td>0</td>\n",
       "      <td>200.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2019-01-01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>50.0</td>\n",
       "      <td>365</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32346</th>\n",
       "      <td>25314051</td>\n",
       "      <td>Large Boerum Hill 2 Bedroom</td>\n",
       "      <td>16284</td>\n",
       "      <td>Laura</td>\n",
       "      <td>1</td>\n",
       "      <td>85</td>\n",
       "      <td>40.68075</td>\n",
       "      <td>-73.98181</td>\n",
       "      <td>0</td>\n",
       "      <td>240.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2018-10-21</td>\n",
       "      <td>0.31</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.487791</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3984</th>\n",
       "      <td>2488227</td>\n",
       "      <td>COZY 1 BR IN COOL LOWER EAST SIDE</td>\n",
       "      <td>13028</td>\n",
       "      <td>Asaf</td>\n",
       "      <td>2</td>\n",
       "      <td>119</td>\n",
       "      <td>40.72005</td>\n",
       "      <td>-73.98609</td>\n",
       "      <td>0</td>\n",
       "      <td>180.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>55.0</td>\n",
       "      <td>2019-06-15</td>\n",
       "      <td>0.85</td>\n",
       "      <td>1.0</td>\n",
       "      <td>97</td>\n",
       "      <td>1</td>\n",
       "      <td>0.464370</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41249</th>\n",
       "      <td>32102590</td>\n",
       "      <td>Cute Studio in Central of Midtown</td>\n",
       "      <td>34281</td>\n",
       "      <td>Studioplus</td>\n",
       "      <td>2</td>\n",
       "      <td>127</td>\n",
       "      <td>40.74670</td>\n",
       "      <td>-73.98706</td>\n",
       "      <td>0</td>\n",
       "      <td>220.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2019-06-18</td>\n",
       "      <td>1.09</td>\n",
       "      <td>13.0</td>\n",
       "      <td>283</td>\n",
       "      <td>1</td>\n",
       "      <td>0.060371</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34388</th>\n",
       "      <td>27277569</td>\n",
       "      <td>LUXURY SUTTON PLACE 2BR 2BATH-DOORMAN/GYM</td>\n",
       "      <td>33868</td>\n",
       "      <td>Pranjal</td>\n",
       "      <td>2</td>\n",
       "      <td>127</td>\n",
       "      <td>40.75375</td>\n",
       "      <td>-73.96362</td>\n",
       "      <td>0</td>\n",
       "      <td>350.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2019-01-01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>65.0</td>\n",
       "      <td>364</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20179</th>\n",
       "      <td>16157387</td>\n",
       "      <td>HUGE BEAUTIFUL 3 BEDROOMS APARTMENT WITH ALL</td>\n",
       "      <td>9999</td>\n",
       "      <td>Maxime</td>\n",
       "      <td>2</td>\n",
       "      <td>94</td>\n",
       "      <td>40.81588</td>\n",
       "      <td>-73.93984</td>\n",
       "      <td>0</td>\n",
       "      <td>400.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2018-09-25</td>\n",
       "      <td>0.10</td>\n",
       "      <td>4.0</td>\n",
       "      <td>180</td>\n",
       "      <td>1</td>\n",
       "      <td>0.457014</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29931</th>\n",
       "      <td>23040052</td>\n",
       "      <td>Cozy 2 bed 2 bath Pool in building</td>\n",
       "      <td>29728</td>\n",
       "      <td>Yaacov</td>\n",
       "      <td>2</td>\n",
       "      <td>201</td>\n",
       "      <td>40.77717</td>\n",
       "      <td>-73.95131</td>\n",
       "      <td>0</td>\n",
       "      <td>300.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2018-12-27</td>\n",
       "      <td>0.49</td>\n",
       "      <td>33.0</td>\n",
       "      <td>342</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2505</th>\n",
       "      <td>1284776</td>\n",
       "      <td>Modern Apartment in Brownstone Bkln</td>\n",
       "      <td>9312</td>\n",
       "      <td>Angus</td>\n",
       "      <td>1</td>\n",
       "      <td>31</td>\n",
       "      <td>40.68013</td>\n",
       "      <td>-73.99397</td>\n",
       "      <td>0</td>\n",
       "      <td>250.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>2019-01-02</td>\n",
       "      <td>0.67</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.425151</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25316</th>\n",
       "      <td>20243875</td>\n",
       "      <td>Bright 3 floors home in Bushwick with precious...</td>\n",
       "      <td>9105</td>\n",
       "      <td>Maria Veronica</td>\n",
       "      <td>1</td>\n",
       "      <td>28</td>\n",
       "      <td>40.70089</td>\n",
       "      <td>-73.92730</td>\n",
       "      <td>1</td>\n",
       "      <td>250.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2019-01-01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>260</td>\n",
       "      <td>1</td>\n",
       "      <td>0.475063</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1511</th>\n",
       "      <td>683681</td>\n",
       "      <td>Beautiful West Village 1 BR apartment</td>\n",
       "      <td>2854</td>\n",
       "      <td>Jeff</td>\n",
       "      <td>2</td>\n",
       "      <td>209</td>\n",
       "      <td>40.73477</td>\n",
       "      <td>-74.00275</td>\n",
       "      <td>0</td>\n",
       "      <td>179.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>2019-06-08</td>\n",
       "      <td>0.46</td>\n",
       "      <td>1.0</td>\n",
       "      <td>284</td>\n",
       "      <td>1</td>\n",
       "      <td>0.499497</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38734</th>\n",
       "      <td>30389336</td>\n",
       "      <td>Bespoke Studio in Hip East Village w/ Gym, Doo...</td>\n",
       "      <td>29407</td>\n",
       "      <td>Blueground</td>\n",
       "      <td>2</td>\n",
       "      <td>64</td>\n",
       "      <td>40.73186</td>\n",
       "      <td>-73.98761</td>\n",
       "      <td>0</td>\n",
       "      <td>281.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2019-01-01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>232.0</td>\n",
       "      <td>322</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33796</th>\n",
       "      <td>26780658</td>\n",
       "      <td>Cozy gem in Greenwich Village @ border of SoHo</td>\n",
       "      <td>19045</td>\n",
       "      <td>Christopher</td>\n",
       "      <td>2</td>\n",
       "      <td>92</td>\n",
       "      <td>40.72704</td>\n",
       "      <td>-73.99672</td>\n",
       "      <td>0</td>\n",
       "      <td>199.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2019-04-23</td>\n",
       "      <td>0.82</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.492250</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1704</th>\n",
       "      <td>772362</td>\n",
       "      <td>TIMES SQ/THEATRE DIST STUDIO</td>\n",
       "      <td>4762</td>\n",
       "      <td>Frank</td>\n",
       "      <td>2</td>\n",
       "      <td>95</td>\n",
       "      <td>40.76078</td>\n",
       "      <td>-73.99076</td>\n",
       "      <td>0</td>\n",
       "      <td>165.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>276.0</td>\n",
       "      <td>2019-07-06</td>\n",
       "      <td>3.44</td>\n",
       "      <td>1.0</td>\n",
       "      <td>146</td>\n",
       "      <td>1</td>\n",
       "      <td>0.058846</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1787</th>\n",
       "      <td>803778</td>\n",
       "      <td>Luxury Loft Noho New York City</td>\n",
       "      <td>6726</td>\n",
       "      <td>Jenny</td>\n",
       "      <td>2</td>\n",
       "      <td>144</td>\n",
       "      <td>40.72591</td>\n",
       "      <td>-73.99452</td>\n",
       "      <td>0</td>\n",
       "      <td>465.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>47.0</td>\n",
       "      <td>2019-05-31</td>\n",
       "      <td>0.67</td>\n",
       "      <td>1.0</td>\n",
       "      <td>277</td>\n",
       "      <td>1</td>\n",
       "      <td>0.406986</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             id                                               name  host_id  \\\n",
       "2095     949713  Luxury Duplex Loft w. Private Roof & Amazing V...     2641   \n",
       "7752    5873714                East Village -  Beautiful 1 Bedroom    19637   \n",
       "45063  34511584  Fifth Avenue Ultra Luxurious Large 3 Bed- Gym/...    26073   \n",
       "43019  33400202  Four Points by Sheraton New York Downtown-7 Ni...     7694   \n",
       "31743  24752747     Sonder | Hanover Square | Simple 1BR + Rooftop    12806   \n",
       "1089     457829    Beautiful apartment in the heart of The Village     4222   \n",
       "33358  26357145               Nice one bedroom near United Nations    30110   \n",
       "32346  25314051                        Large Boerum Hill 2 Bedroom    16284   \n",
       "3984    2488227                  COZY 1 BR IN COOL LOWER EAST SIDE    13028   \n",
       "41249  32102590                  Cute Studio in Central of Midtown    34281   \n",
       "34388  27277569          LUXURY SUTTON PLACE 2BR 2BATH-DOORMAN/GYM    33868   \n",
       "20179  16157387       HUGE BEAUTIFUL 3 BEDROOMS APARTMENT WITH ALL     9999   \n",
       "29931  23040052                 Cozy 2 bed 2 bath Pool in building    29728   \n",
       "2505    1284776                Modern Apartment in Brownstone Bkln     9312   \n",
       "25316  20243875  Bright 3 floors home in Bushwick with precious...     9105   \n",
       "1511     683681              Beautiful West Village 1 BR apartment     2854   \n",
       "38734  30389336  Bespoke Studio in Hip East Village w/ Gym, Doo...    29407   \n",
       "33796  26780658     Cozy gem in Greenwich Village @ border of SoHo    19045   \n",
       "1704     772362                       TIMES SQ/THEATRE DIST STUDIO     4762   \n",
       "1787     803778                     Luxury Loft Noho New York City     6726   \n",
       "\n",
       "            host_name  neighbourhood_group  neighbourhood  latitude  \\\n",
       "2095              Tal                    1             85  40.67450   \n",
       "7752           Majeed                    2             64  40.73182   \n",
       "45063            John                    2            127  40.76261   \n",
       "43019             Yun                    2             73  40.70832   \n",
       "31743          Sonder                    2             73  40.70518   \n",
       "1089           Lorena                    2             64  40.72917   \n",
       "33358         Stanley                    2            137  40.75030   \n",
       "32346           Laura                    1             85  40.68075   \n",
       "3984             Asaf                    2            119  40.72005   \n",
       "41249      Studioplus                    2            127  40.74670   \n",
       "34388         Pranjal                    2            127  40.75375   \n",
       "20179          Maxime                    2             94  40.81588   \n",
       "29931          Yaacov                    2            201  40.77717   \n",
       "2505            Angus                    1             31  40.68013   \n",
       "25316  Maria Veronica                    1             28  40.70089   \n",
       "1511             Jeff                    2            209  40.73477   \n",
       "38734      Blueground                    2             64  40.73186   \n",
       "33796     Christopher                    2             92  40.72704   \n",
       "1704            Frank                    2             95  40.76078   \n",
       "1787            Jenny                    2            144  40.72591   \n",
       "\n",
       "       longitude  room_type  price  minimum_nights  number_of_reviews  \\\n",
       "2095   -73.99582          0  400.0            30.0               30.0   \n",
       "7752   -73.98450          1  195.0             5.0               19.0   \n",
       "45063  -73.97680          0  560.0            30.0                0.0   \n",
       "43019  -74.00755          1  169.0             7.0                1.0   \n",
       "31743  -74.00994          0  182.0            29.0                2.0   \n",
       "1089   -73.98811          0  210.0             1.0              178.0   \n",
       "33358  -73.97605          0  200.0            30.0                0.0   \n",
       "32346  -73.98181          0  240.0             3.0                4.0   \n",
       "3984   -73.98609          0  180.0             6.0               55.0   \n",
       "41249  -73.98706          0  220.0             2.0                4.0   \n",
       "34388  -73.96362          0  350.0            30.0                0.0   \n",
       "20179  -73.93984          0  400.0             2.0                3.0   \n",
       "29931  -73.95131          0  300.0            31.0                8.0   \n",
       "2505   -73.99397          0  250.0             5.0               49.0   \n",
       "25316  -73.92730          1  250.0             2.0                0.0   \n",
       "1511   -74.00275          0  179.0            14.0               37.0   \n",
       "38734  -73.98761          0  281.0            90.0                0.0   \n",
       "33796  -73.99672          0  199.0             2.0                9.0   \n",
       "1704   -73.99076          0  165.0             3.0              276.0   \n",
       "1787   -73.99452          0  465.0            30.0               47.0   \n",
       "\n",
       "      last_review  reviews_per_month  calculated_host_listings_count  \\\n",
       "2095   2019-06-01               0.39                             1.0   \n",
       "7752   2016-11-28               0.37                             1.0   \n",
       "45063  2019-01-01               0.00                             1.0   \n",
       "43019  2019-05-17               0.57                            13.0   \n",
       "31743  2019-04-01               0.24                            96.0   \n",
       "1089   2019-07-01               2.06                             1.0   \n",
       "33358  2019-01-01               0.00                            50.0   \n",
       "32346  2018-10-21               0.31                             1.0   \n",
       "3984   2019-06-15               0.85                             1.0   \n",
       "41249  2019-06-18               1.09                            13.0   \n",
       "34388  2019-01-01               0.00                            65.0   \n",
       "20179  2018-09-25               0.10                             4.0   \n",
       "29931  2018-12-27               0.49                            33.0   \n",
       "2505   2019-01-02               0.67                             1.0   \n",
       "25316  2019-01-01               0.00                             1.0   \n",
       "1511   2019-06-08               0.46                             1.0   \n",
       "38734  2019-01-01               0.00                           232.0   \n",
       "33796  2019-04-23               0.82                             3.0   \n",
       "1704   2019-07-06               3.44                             1.0   \n",
       "1787   2019-05-31               0.67                             1.0   \n",
       "\n",
       "       availability_365  target   predict  predround  deltaval  \n",
       "2095                 67       1  0.353480          0         1  \n",
       "7752                178       1  0.485379          0         1  \n",
       "45063               337       1  0.495361          0         1  \n",
       "43019               358       1  0.008758          0         1  \n",
       "31743               347       1  0.000000          0         1  \n",
       "1089                  0       1  0.239129          0         1  \n",
       "33358               365       1  0.000000          0         1  \n",
       "32346                 0       1  0.487791          0         1  \n",
       "3984                 97       1  0.464370          0         1  \n",
       "41249               283       1  0.060371          0         1  \n",
       "34388               364       1  0.000000          0         1  \n",
       "20179               180       1  0.457014          0         1  \n",
       "29931               342       1  0.000000          0         1  \n",
       "2505                  0       1  0.425151          0         1  \n",
       "25316               260       1  0.475063          0         1  \n",
       "1511                284       1  0.499497          0         1  \n",
       "38734               322       1  0.000000          0         1  \n",
       "33796                 0       1  0.492250          0         1  \n",
       "1704                146       1  0.058846          0         1  \n",
       "1787                277       1  0.406986          0         1  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get subset of dataframe with wrong guesses\n",
    "# k1 = df.loc[(df.Product == p_id)\n",
    "testwrong = test.loc[(test.deltaval == 1)]\n",
    "testright = test.loc[(test.deltaval == 0)]\n",
    "testwrong.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x22367711788>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAWN0lEQVR4nO3df5BddX3G8ffTUJnIyg8NrDHBBttgSxLNuGtKbWF2BywRsIFWaygjROysMOjUaewArSOMTGYyrakdQGCiYYCBsjJFTARiRWSLrUkhYSKbANEAW9gkkwyGJixmUjd8+sc9K9flbvbsuXfPvez3ec3c2Xu/53vO+Zxz7z577vm1igjMzCwNv9XsAszMrDwOfTOzhDj0zcwS4tA3M0uIQ9/MLCFHNbuA8cyYMSPmzJlTaNzXXnuNY445prEFNZhrrF+r1weusVFcY36bN29+OSJOfNOAiGjpR0dHRxT16KOPFh63LK6xfq1eX4RrbBTXmB+wKWpkqnfvmJklxKFvZpaQcUNf0m2S9kraWtX2bUlbsseApC1Z+xxJB6uG3Vo1Toekfkk7JN0gSZOzSGZmNpY8B3JvB24C7hxpiIhPjTyXtArYX9X/uYhYWGM6twA9wEbgIWAxsH7iJZuZWVHjbulHxGPAvlrDsq31vwTuOdI0JM0Ejo2IDdkBhjuBCyZerpmZ1UOR44ZrkuYAD0TE/FHtZwL/HBGdVf22AT8DDgBfjogfS+oEVkbE2Vm/M4CrIuL8MebXQ+VbAe3t7R29vb1Flo2hoSHa2toKjVsW11i/Vq8PXGOjuMb8uru7N49k82+odUrP6AcwB9hao/0WYHnV66OBd2XPO4CXgGOBDwM/rOp3BvC9PPP2KZvN1+o1tnp9Ea6xUVxjfoxxymbhi7MkHQX8eRbuI39ADgGHsuebJT0HnAoMArOrRp8N7Co6bzMzK6aeUzbPBp6NiMGRBkknSpqWPX8fMBd4PiJ2A69KOj07DnAJsLaOeZuZWQHjbulLugfoAmZIGgSujYg1wFLefAD3TOCrkoaBw8DlETFyEPgKKmcCTady1o7P3JlC5lz9YNPmffvi5l/ybvZWMW7oR8RFY7Qvq9F2H3DfGP03AfNrDTMzs3L4ilwzs4Q49M3MEuLQNzNLSMvfT99sPP0797OsCQeSB1aeV/o8zerlLX0zs4Q49M3MEuLQNzNLiEPfzCwhDn0zs4Q49M3MEuLQNzNLiEPfzCwhDn0zs4Q49M3MEuLQNzNLiEPfzCwhDn0zs4Q49M3MEuLQNzNLiEPfzCwhDn0zs4SMG/qSbpO0V9LWqrbrJO2UtCV7nFs17BpJOyRtl3ROVXuHpP5s2A2S1PjFMTOzI8mzpX87sLhG+9cjYmH2eAhA0mnAUmBeNs7NkqZl/W8BeoC52aPWNM3MbBKNG/oR8RiwL+f0lgC9EXEoIl4AdgCLJM0Ejo2IDRERwJ3ABUWLNjOzYlTJ4HE6SXOAByJifvb6OmAZcADYBCyPiFck3QRsjIi7sn5rgPXAALAyIs7O2s8AroqI88eYXw+VbwW0t7d39Pb2Flq4oaEh2traCo1blqlSY//O/SVV82bt02HPwfLnu2DWcbn7TpX3udlcY37d3d2bI6JzdPtRBad3C3A9ENnPVcBlQK399HGE9poiYjWwGqCzszO6uroKFdnX10fRccsyVWpcdvWD5RRTw/IFw6zqL/pRLm7g4q7cfafK+9xsrrF+hc7eiYg9EXE4Il4HvgksygYNAidXdZ0N7MraZ9doNzOzEhUK/Wwf/YgLgZEze9YBSyUdLekUKgdsH4+I3cCrkk7Pztq5BFhbR91mZlbAuN+JJd0DdAEzJA0C1wJdkhZS2UUzAHwOICK2SboXeBoYBq6MiMPZpK6gcibQdCr7+dc3ckHMzGx844Z+RFxUo3nNEfqvAFbUaN8EzJ9QdWZm1lC+ItfMLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCHjhr6k2yTtlbS1qu2fJD0r6SlJ90s6PmufI+mgpC3Z49aqcTok9UvaIekGSZqcRTIzs7Hk2dK/HVg8qu1hYH5EfAD4GXBN1bDnImJh9ri8qv0WoAeYmz1GT9PMzCbZuKEfEY8B+0a1/SAihrOXG4HZR5qGpJnAsRGxISICuBO4oFjJZmZWlCoZPE4naQ7wQETMrzHse8C3I+KurN82Klv/B4AvR8SPJXUCKyPi7GycM4CrIuL8MebXQ+VbAe3t7R29vb0TXzJgaGiItra2QuOWZarU2L9zf0nVvFn7dNhzsPz5Lph1XO6+U+V9bjbXmF93d/fmiOgc3X5UPROV9A/AMHB31rQbeG9E/EJSB/BdSfOAWvvvx/xrExGrgdUAnZ2d0dXVVai+vr4+io5blqlS47KrHyynmBqWLxhmVX9dH+VCBi7uyt13qrzPzeYa61f4N0XSpcD5wFnZLhsi4hBwKHu+WdJzwKnAIL+5C2g2sKvovM3MrJhCp2xKWgxcBfxZRPyyqv1ESdOy5++jcsD2+YjYDbwq6fTsrJ1LgLV1V29mZhMy7pa+pHuALmCGpEHgWipn6xwNPJydebkxO1PnTOCrkoaBw8DlETFyEPgKKmcCTQfWZw8zMyvRuKEfERfVaF4zRt/7gPvGGLYJeNOBYDMzK4+vyDUzS4hD38wsIQ59M7OEOPTNzBLi0DczS4hD38wsIQ59M7OEOPTNzBLi0DczS4hD38wsIQ59M7OEOPTNzBLi0DczS4hD38wsIQ59M7OElP+PRUvUv3N/U/5368DK80qfp5lZHt7SNzNLiEPfzCwhDn0zs4Q49M3MEuLQNzNLyLihL+k2SXslba1qe6ekhyX9PPt5QtWwayTtkLRd0jlV7R2S+rNhN0hS4xfHzMyOJM+W/u3A4lFtVwOPRMRc4JHsNZJOA5YC87JxbpY0LRvnFqAHmJs9Rk/TzMwm2bihHxGPAftGNS8B7sie3wFcUNXeGxGHIuIFYAewSNJM4NiI2BARAdxZNY6ZmZVElQwep5M0B3ggIuZnr/83Io6vGv5KRJwg6SZgY0TclbWvAdYDA8DKiDg7az8DuCoizh9jfj1UvhXQ3t7e0dvbW2jh9u7bz56DhUaty4JZx+XuOzQ0RFtb2yRWU788Nfbv3F9SNW/WPh2/zw3gGhujVWrs7u7eHBGdo9sbfUVurf30cYT2miJiNbAaoLOzM7q6ugoVc+Pda1nVX/5FxwMXd+Xu29fXR9HlK0ueGptx5fOI5QuG/T43gGtsjFavsejZO3uyXTZkP/dm7YPAyVX9ZgO7svbZNdrNzKxERUN/HXBp9vxSYG1V+1JJR0s6hcoB28cjYjfwqqTTs7N2Lqkax8zMSjLud2JJ9wBdwAxJg8C1wErgXkmfBV4EPgkQEdsk3Qs8DQwDV0bE4WxSV1A5E2g6lf386xu6JGZmNq5xQz8iLhpj0Flj9F8BrKjRvgmYP6HqzMysoXxFrplZQhz6ZmYJceibmSXEoW9mlhCHvplZQhz6ZmYJceibmSXEoW9mlhCHvplZQhz6ZmYJceibmSXEoW9mlpDy//OEmb1lzWnSP8sZWHleU+Y7FXlL38wsIQ59M7OEOPTNzBLi0DczS4hD38wsIQ59M7OEOPTNzBLi0DczS0jh0Jf0fklbqh4HJH1R0nWSdla1n1s1zjWSdkjaLumcxiyCmZnlVfiK3IjYDiwEkDQN2AncD3wG+HpEfK26v6TTgKXAPOA9wA8lnRoRh4vWYGZmE9Oo2zCcBTwXEf8jaaw+S4DeiDgEvCBpB7AI2NCgGozJuUx++YJhljXp8nsza6xG7dNfCtxT9frzkp6SdJukE7K2WcBLVX0GszYzMyuJIqK+CUhvA3YB8yJij6R24GUggOuBmRFxmaRvABsi4q5svDXAQxFxX41p9gA9AO3t7R29vb2Fatu7bz97DhYatS4LZh2Xu+/Q0BBtbW0Nm3f/zv0Nm9aI9uk0ZT3m1az6mvk+T4Y8NU7G5yuPkXU9VdZjGbq7uzdHROfo9kbs3vkY8GRE7AEY+Qkg6ZvAA9nLQeDkqvFmU/lj8SYRsRpYDdDZ2RldXV2FCrvx7rWs6i//RqIDF3fl7tvX10fR5atlMnbDLF8w3JT1mFez6mvm+zwZ8tTYrN18I+t6qqzHZmrE7p2LqNq1I2lm1bALga3Z83XAUklHSzoFmAs83oD5m5lZTnVtHkl6O/BR4HNVzf8oaSGV3TsDI8MiYpuke4GngWHgSp+5Y2ZWrrpCPyJ+CbxrVNunj9B/BbCinnmamVlxviLXzCwhDn0zs4Q49M3MEuLQNzNLiEPfzCwhDn0zs4Q49M3MEuLQNzNLiEPfzCwhDn0zs4Q49M3MEuLQNzNLiEPfzCwhDn0zs4Q49M3MEuLQNzNLiEPfzCwhrfvfrs1a3JwJ/JPw5QuGG/pPxQdWntewaVlavKVvZpYQh76ZWUIc+mZmCakr9CUNSOqXtEXSpqztnZIelvTz7OcJVf2vkbRD0nZJ59RbvJmZTUwjtvS7I2JhRHRmr68GHomIucAj2WsknQYsBeYBi4GbJU1rwPzNzCynydi9swS4I3t+B3BBVXtvRByKiBeAHcCiSZi/mZmNod7QD+AHkjZL6sna2iNiN0D286SsfRbwUtW4g1mbmZmVRBFRfGTpPRGxS9JJwMPAF4B1EXF8VZ9XIuIESd8ANkTEXVn7GuChiLivxnR7gB6A9vb2jt7e3kL17d23nz0HC41alwWzjsvdd2hoiLa2tobNu3/n/oZNa0T7dJqyHvNq9fqg8TVO5DOWV57P4mR8vvIYWd5G/75Mhlapsbu7e3PVbvdfq+virIjYlf3cK+l+Krtr9kiaGRG7Jc0E9mbdB4GTq0afDewaY7qrgdUAnZ2d0dXVVai+G+9ey6r+8q8/G7i4K3ffvr4+ii5fLY28AGjE8gXDTVmPebV6fdD4GifyGcsrz2dxMj5feYwsb6N/XyZDq9dYePeOpGMkvWPkOfCnwFZgHXBp1u1SYG32fB2wVNLRkk4B5gKPF52/mZlNXD2bHu3A/ZJGpvOvEfF9SU8A90r6LPAi8EmAiNgm6V7gaWAYuDIiDtdVvZmZTUjh0I+I54EP1mj/BXDWGOOsAFYUnaeZVUzkvj95Nfr+QNaafEWumVlCHPpmZglx6JuZJcShb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWkNa+YclbVDP/YbaZ2ZF4S9/MLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCG+ItfMWt7IVe7NuIJ9YOV5pc5vsnlL38wsIQ59M7OEOPTNzBJSOPQlnSzpUUnPSNom6W+y9usk7ZS0JXucWzXONZJ2SNou6ZxGLICZmeVXz4HcYWB5RDwp6R3AZkkPZ8O+HhFfq+4s6TRgKTAPeA/wQ0mnRsThOmowM7MJKLylHxG7I+LJ7PmrwDPArCOMsgTojYhDEfECsANYVHT+ZmY2cYqI+icizQEeA+YDfwssAw4Am6h8G3hF0k3Axoi4KxtnDbA+Iv6txvR6gB6A9vb2jt7e3kJ17d23nz0HC41amvbpuMY6tXp94BobpRk1Lph13IT6Dw0N0dbWNknV5Nfd3b05IjpHt9d9nr6kNuA+4IsRcUDSLcD1QGQ/VwGXAaoxes2/OBGxGlgN0NnZGV1dXYVqu/Hutazqb+1LEZYvGHaNdWr1+sA1Nkozahy4uGtC/fv6+iiaWWWo6+wdSb9NJfDvjojvAETEnog4HBGvA9/kjV04g8DJVaPPBnbVM38zM5uYes7eEbAGeCYi/rmqfWZVtwuBrdnzdcBSSUdLOgWYCzxedP5mZjZx9XxP+mPg00C/pC1Z298DF0laSGXXzQDwOYCI2CbpXuBpKmf+XOkzd8zMylU49CPiP6m9n/6hI4yzAlhRdJ5mZlYfX5FrZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpaQ0kNf0mJJ2yXtkHR12fM3M0vZUWXOTNI04BvAR4FB4AlJ6yLi6TLrMDPLa87VD06o//IFwyyb4Di1DKw8r+5p1FL2lv4iYEdEPB8R/wf0AktKrsHMLFmKiPJmJn0CWBwRf529/jTwhxHx+VH9eoCe7OX7ge0FZzkDeLnguGVxjfVr9frANTaKa8zvdyLixNGNpe7eAVSj7U1/dSJiNbC67plJmyKis97pTCbXWL9Wrw9cY6O4xvqVvXtnEDi56vVsYFfJNZiZJavs0H8CmCvpFElvA5YC60quwcwsWaXu3omIYUmfB/4dmAbcFhHbJnGWde8iKoFrrF+r1weusVFcY51KPZBrZmbN5StyzcwS4tA3M0vIlAj98W7toIobsuFPSfpQi9X3+5I2SDok6Utl1jaBGi/O1t1Tkn4i6YMtWOOSrL4tkjZJ+pNWq7Gq34clHc6uXSlVjvXYJWl/th63SPpKq9VYVecWSdsk/Ucr1Sfp76rW39bsvX5nmTWOKSLe0g8qB4SfA94HvA34KXDaqD7nAuupXCdwOvDfLVbfScCHgRXAl1p0HX4EOCF7/rEy1+EEamzjjeNUHwCebbUaq/r9CHgI+ESr1Qh0AQ+U/TmcYI3HA08D781en9RK9Y3q/3HgR81an6MfU2FLP8+tHZYAd0bFRuB4STNbpb6I2BsRTwC/Kqmm0fLU+JOIeCV7uZHKNRatVuNQZL9lwDHUuPCv2TVmvgDcB+wts7jMW+FWKHlq/CvgOxHxIlR+h1qsvmoXAfeUUlkOUyH0ZwEvVb0ezNom2meyNHPeeU20xs9S+eZUplw1SrpQ0rPAg8BlJdU2YtwaJc0CLgRuLbGuannf6z+S9FNJ6yXNK6e0X8tT46nACZL6JG2WdElp1U3g90XS24HFVP7It4Syb8MwGfLc2iHX7R8mSTPnnVfuGiV1Uwn9sveX572Fx/3A/ZLOBK4Hzp7swqrkqfFfgKsi4rBUq/uky1Pjk1Tu2zIk6Vzgu8DcSa/sDXlqPAroAM4CpgMbJG2MiJ9NdnFM7Hf648B/RcS+SaxnQqZC6Oe5tUMzb//wVrj1RK4aJX0A+BbwsYj4RUm1jZjQeoyIxyT9rqQZEVHWza/y1NgJ9GaBPwM4V9JwRHy3nBLHrzEiDlQ9f0jSzS24HgeBlyPiNeA1SY8BHwTKCP2JfBaX0kK7doApcSD3KOB54BTeOKgyb1Sf8/jNA7mPt1J9VX2vozkHcvOsw/cCO4CPtPD7/Hu8cSD3Q8DOkdetUuOo/rdT/oHcPOvx3VXrcRHwYqutR+APgEeyvm8HtgLzW6W+rN9xwD7gmDLf4/Eeb/kt/Rjj1g6SLs+G30rlLIlzqYTWL4HPtFJ9kt4NbAKOBV6X9EUqZwMcGHPCJdcIfAV4F3BztpU6HCXeSTBnjX8BXCLpV8BB4FOR/fa1UI1NlbPGTwBXSBqmsh6Xttp6jIhnJH0feAp4HfhWRGxtlfqyrhcCP4jKt5GW4dswmJklZCqcvWNmZjk59M3MEuLQNzNLiEPfzCwhDn0zs4Q49M3MEuLQNzNLyP8D4fJWva682a4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# dvalid.hist(range = (0,5))\n",
    "# apar_ds.Time_to_relief.hist(range = (0,5))\n",
    "dvalid.predict.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x223677a13c8>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAARc0lEQVR4nO3df5DcdX3H8edbUp1IBgiNnGmCDW3T1kDUMSelWjuXwSkR6gSmOg1lBCqdtAx26kxsDf1DnTqZ5h9sBxVtKg4w/rjJVBFGiC0TuTJWKCYOEgJSU8nEECYZhEaOMtTDd//YL7Ice7m9vd3v7t7n+ZjZud3Pfn+89rt3r/ved3e/F5mJJKkMr+p3AElSfSx9SSqIpS9JBbH0Jakglr4kFWRRvwPMZtmyZblq1aqO5n322Wc5+eSTuxuoB4YlJwxPVnN237BkHZac0Nuse/fufTIzX/eKOzJzoC/r1q3LTt19990dz1unYcmZOTxZzdl9w5J1WHJm9jYrsCdbdKqHdySpIJa+JBXE0pekglj6klQQS1+SCmLpS1JBLH1JKoilL0kFsfQlqSADfxoGaTartt7xirEta6e4ssV4Nx3cflFPly/1gnv6klQQS1+SCmLpS1JBLH1JKoilL0kFsfQlqSCWviQVxNKXpIJY+pJUEEtfkgpi6UtSQSx9SSqIpS9JBbH0Jakglr4kFcTSl6SCWPqSVBBLX5IKMmvpR8SZEXF3RDwSEfsj4q+q8dMj4q6I+GH1dWnTPNdGxIGIeDQiLmgaXxcR+6r7ro+I6M3DkiS10s6e/hSwJTPfCJwHXBMRa4CtwO7MXA3srm5T3bcJOBvYANwQESdVy/ossBlYXV02dPGxSJJmMWvpZ+YTmfm96vozwCPACmAjcHM12c3AxdX1jcB4Zj6fmY8BB4BzI2I5cEpm3puZCdzSNI8kqQbR6N82J45YBdwDnAMcyszTmu57OjOXRsSngfsy84vV+I3ALuAgsD0z31WNvxP4SGb+YYv1bKbxFwEjIyPrxsfHO3pwk5OTLFmypKN56zQsOWEws+57/PgrxkYWw9HnervetStOnfcyBnF7zmRYsg5LTuht1vXr1+/NzNHp44vaXUBELAG+CnwoM396gsPxre7IE4y/cjBzB7ADYHR0NMfGxtqN+TITExN0Om+dhiUnDGbWK7fe8YqxLWunuG5f29/eHTl42di8lzGI23Mmw5J1WHJCf7K29e6diPglGoX/pcz8WjV8tDpkQ/X1WDV+GDizafaVwJFqfGWLcUlSTdp5904ANwKPZOYnm+66Hbiiun4FcFvT+KaIeE1EnEXjBdv7M/MJ4JmIOK9a5uVN80iSatDO37/vAN4P7IuIB6qxvwW2Azsj4irgEPA+gMzcHxE7gYdpvPPnmsx8oZrvauAmYDGN4/y7uvQ4JEltmLX0M/PbtD4eD3D+DPNsA7a1GN9D40VgSVIf+IlcSSqIpS9JBbH0Jakglr4kFcTSl6SCWPqSVBBLX5IKYulLUkEsfUkqSG9PQ6hirGpxpktJg8c9fUkqiKUvSQWx9CWpIJa+JBXE0pekglj6klQQS1+SCmLpS1JBLH1JKoilL0kFsfQlqSCWviQVxNKXpIJY+pJUEEtfkgpi6UtSQSx9SSqIpS9JBbH0Jakglr4kFcTSl6SCWPqSVBBLX5IKYulLUkEsfUkqiKUvSQWx9CWpILOWfkR8ISKORcRDTWMfj4jHI+KB6nJh033XRsSBiHg0Ii5oGl8XEfuq+66PiOj+w5EknUg7e/o3ARtajP9DZr6lutwJEBFrgE3A2dU8N0TESdX0nwU2A6urS6tlSpJ6aNbSz8x7gKfaXN5GYDwzn8/Mx4ADwLkRsRw4JTPvzcwEbgEu7jS0JKkz8zmm/8GIeLA6/LO0GlsB/LhpmsPV2Irq+vRxSVKNorHjPctEEauAb2TmOdXtEeBJIIFPAMsz8wMR8Rng3sz8YjXdjcCdwCHg7zPzXdX4O4G/ycz3zLC+zTQOBTEyMrJufHy8owc3OTnJkiVLOpq3TsOSE2bOuu/x431IM7ORxXD0ud6uY+2KU+e9jIXw3A+aYckJvc26fv36vZk5On18UScLy8yjL16PiH8GvlHdPAyc2TTpSuBINb6yxfhMy98B7AAYHR3NsbGxTmIyMTFBp/PWaVhywsxZr9x6R/1hTmDL2imu29fRt3fbDl42Nu9lLITnftAMS07oT9aODu9Ux+hfdAnw4jt7bgc2RcRrIuIsGi/Y3p+ZTwDPRMR51bt2Lgdum0duSVIHZt0VioivAGPAsog4DHwMGIuIt9A4vHMQ+HOAzNwfETuBh4Ep4JrMfKFa1NU03gm0GNhVXSRJNZq19DPz0hbDN55g+m3Athbje4Bz5pROktRVfiJXkgpi6UtSQSx9SSqIpS9JBbH0Jakglr4kFcTSl6SCWPqSVBBLX5IK0tszUvXZvseP9+VEYAe3X1T7OiWpHe7pS1JBLH1JKoilL0kFsfQlqSCWviQVxNKXpIJY+pJUEEtfkgpi6UtSQSx9SSqIpS9JBbH0Jakglr4kFcTSl6SCWPqSVBBLX5IKYulLUkEsfUkqiKUvSQWx9CWpIJa+JBXE0pekglj6klQQS1+SCmLpS1JBLH1JKoilL0kFmbX0I+ILEXEsIh5qGjs9Iu6KiB9WX5c23XdtRByIiEcj4oKm8XURsa+67/qIiO4/HEnSibSzp38TsGHa2FZgd2auBnZXt4mINcAm4Oxqnhsi4qRqns8Cm4HV1WX6MiVJPTZr6WfmPcBT04Y3AjdX128GLm4aH8/M5zPzMeAAcG5ELAdOycx7MzOBW5rmkSTVpNNj+iOZ+QRA9fWManwF8OOm6Q5XYyuq69PHJUk1WtTl5bU6Tp8nGG+9kIjNNA4FMTIywsTEREdhRhbDlrVTHc07H3PNOzk52fFjrNtMWfuxnU+kjue+G8/ZQnjuB82w5IT+ZO209I9GxPLMfKI6dHOsGj8MnNk03UrgSDW+ssV4S5m5A9gBMDo6mmNjYx2F/NSXbuO6fd3+vTa7g5eNzWn6iYkJOn2MdZsp65Vb76g/zAlsWTvV8+d+rs9zK8P23F/5zWf7su6D2y9qe9ph26Z1Z+308M7twBXV9SuA25rGN0XEayLiLBov2N5fHQJ6JiLOq961c3nTPJKkmsy6KxQRXwHGgGURcRj4GLAd2BkRVwGHgPcBZOb+iNgJPAxMAddk5gvVoq6m8U6gxcCu6iJJqtGspZ+Zl85w1/kzTL8N2NZifA9wzpzSSZK6yk/kSlJBLH1JKoilL0kFsfQlqSCWviQVxNKXpIJY+pJUEEtfkgpi6UtSQeo/G5m0QKzqwknmtqyd6uhkdXM5AZnUzD19SSqIpS9JBfHwzgLTjUMOJ9Lp4QhJg8E9fUkqiKUvSQWx9CWpIJa+JBXE0pekglj6klQQS1+SCmLpS1JB/HCWpIE3lw8ddvsDhAvtPEfu6UtSQSx9SSqIpS9JBbH0Jakglr4kFcTSl6SCWPqSVBBLX5IKYulLUkEsfUkqiKUvSQWx9CWpIJa+JBXE0pekglj6klQQS1+SCjKv0o+IgxGxLyIeiIg91djpEXFXRPyw+rq0afprI+JARDwaERfMN7wkaW66sae/PjPfkpmj1e2twO7MXA3srm4TEWuATcDZwAbghog4qQvrlyS1qReHdzYCN1fXbwYubhofz8znM/Mx4ABwbg/WL0maQWRm5zNHPAY8DSTwT5m5IyL+JzNPa5rm6cxcGhGfBu7LzC9W4zcCuzLzX1osdzOwGWBkZGTd+Ph4R/mOPXWco891NOu8rF1x6pymn5ycZMmSJV1Z977Hj3dlOTMZWUxftulcLfScc/0e64bJyUkeO/5C7eudq24/973c1t382Z9u/fr1e5uOwPzCfP8x+jsy80hEnAHcFRE/OMG00WKs5W+czNwB7AAYHR3NsbGxjsJ96ku3cd2++v/3+8HLxuY0/cTEBJ0+xum6+Q+hW9mydqov23SuFnrOuX6PdcPExATXffvZ2tc7V91+7nu5rbv5s9+ueW2ZzDxSfT0WEbfSOFxzNCKWZ+YTEbEcOFZNfhg4s2n2lcCR+ax/UK2aY/FuWTvV87KWJJhH6UfEycCrMvOZ6vofAH8H3A5cAWyvvt5WzXI78OWI+CTwK8Bq4P55ZJeKNdcdi27YsnaK+R8cUL/N5xkcAW6NiBeX8+XM/GZEfBfYGRFXAYeA9wFk5v6I2Ak8DEwB12Tm4B8glKQFpOPSz8wfAW9uMf4T4PwZ5tkGbOt0nZKk+fETuZJUEEtfkgpi6UtSQSx9SSqIpS9JBbH0Jakglr4kFcTSl6SCWPqSVBBLX5IKYulLUkEsfUkqiKUvSQWx9CWpIJa+JBXE0pekglj6klQQS1+SCmLpS1JBLH1JKoilL0kFsfQlqSCWviQVxNKXpIJY+pJUEEtfkgpi6UtSQSx9SSqIpS9JBbH0Jakglr4kFcTSl6SCWPqSVBBLX5IKsqjfASRpkK3aekfPlr1l7RRXzrD8g9sv6sk63dOXpIJY+pJUkNpLPyI2RMSjEXEgIrbWvX5JKlmtpR8RJwGfAd4NrAEujYg1dWaQpJLVvad/LnAgM3+Umf8HjAMba84gScWKzKxvZRHvBTZk5p9Vt98P/E5mfnDadJuBzdXN3wIe7XCVy4AnO5y3TsOSE4Ynqzm7b1iyDktO6G3WX83M100frPstm9Fi7BW/dTJzB7Bj3iuL2JOZo/NdTq8NS04Ynqzm7L5hyTosOaE/Wes+vHMYOLPp9krgSM0ZJKlYdZf+d4HVEXFWRLwa2ATcXnMGSSpWrYd3MnMqIj4I/CtwEvCFzNzfw1XO+xBRTYYlJwxPVnN237BkHZac0Iestb6QK0nqLz+RK0kFsfQlqSALovRnO7VDNFxf3f9gRLx1QHP+dkTcGxHPR8SH+5GxyjFbzsuq7fhgRHwnIt7cj5xVltmybqxyPhAReyLi9wYxZ9N0b4uIF6rPtPRFG9t0LCKOV9v0gYj46CDmrKYZqzLuj4h/rztjlWG27fnXTdvyoer5P71ngTJzqC80XhD+b+DXgFcD3wfWTJvmQmAXjc8JnAf854DmPAN4G7AN+PAAb8+3A0ur6+/ux/acQ9YlvPTa1ZuAHwxizqbpvgXcCbx3gLfpGPCNfuSbY87TgIeBN1S3zxjEnNOmfw/wrV5mWgh7+u2c2mEjcEs23AecFhHLBy1nZh7LzO8CP6s5W7N2cn4nM5+ubt5H4/MW/dBO1smsfpqAk2nxYcAatHv6kb8EvgocqzPcNMNyqpR2cv4J8LXMPASNn6+aM8Lct+elwFd6GWghlP4K4MdNtw9XY3OdptcGIUM75przKhp/RfVDW1kj4pKI+AFwB/CBmrI1mzVnRKwALgE+V2OuVtp9/n83Ir4fEbsi4ux6or1MOzl/E1gaERMRsTciLq8t3Uva/nmKiNcCG2j84u+ZhfCfs9o5tUNbp3/osUHI0I62c0bEehql35fj5LR/Wo9bgVsj4veBTwDv6nWwadrJ+Y/ARzLzhYhWk9emnazfo3Fel8mIuBD4OrC658lerp2ci4B1wPnAYuDeiLgvM/+r1+GazOXn/j3Af2TmUz3MsyBKv51TOwzC6R8GIUM72soZEW8CPg+8OzN/UlO26ea0TTPznoj49YhYlpl1npCrnZyjwHhV+MuACyNiKjO/Xk/EX5g1a2b+tOn6nRFxw4Bu08PAk5n5LPBsRNwDvBmos/Tn8j26iR4f2gEWxAu5i4AfAWfx0gslZ0+b5iJe/kLu/YOYs2naj9O/F3Lb2Z5vAA4Abx+C5/43eOmF3LcCj794e5ByTpv+Jvr3Qm472/T1Tdv0XODQIG5T4I3A7mra1wIPAecMWs5qulOBp4CTe51p6Pf0c4ZTO0TEX1T3f47GuyEupFFU/wv86SDmjIjXA3uAU4CfR8SHaLzS/9MZF9yHnMBHgV8Gbqj2TKeyD2c1bDPrHwGXR8TPgOeAP87qp2zAcg6ENrO+F7g6IqZobNNNg7hNM/ORiPgm8CDwc+DzmfnQoOWsJr0E+Lds/FXSU56GQZIKshDevSNJapOlL0kFsfQlqSCWviQVxNKXpIJY+pJUEEtfkgry/2xpmPjX+2+IAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# dvalid.hist(range = (0,5))\n",
    "# apar_ds.Time_to_relief.hist(range = (0,5))\n",
    "test.predict.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "925\n",
      "percentage correct validate\n",
      "0.8817740286298569\n"
     ]
    }
   ],
   "source": [
    "# get delta between predictions on training set and actual training target values\n",
    "# hand calculate accuracy on training set as ratio of (total training samples - wrong training predictions)/total training samples\n",
    "\n",
    "deltatr = abs(dvalid.target[:100000] - dvalid.predround[:100000])\n",
    "deltatr[:50]\n",
    "print(deltatr.sum())\n",
    "print(\"percentage correct validate\")\n",
    "print((len(deltatr) - deltatr.sum())/len(deltatr))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['loss', 'accuracy', 'accuracy_1', 'val_loss', 'val_accuracy', 'val_accuracy_1'])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXhU5dnH8e+dDQIEgqyy75soIBFErTsVbalLte7WWkXa2lprqda2b/vWt63WpVVcKcWl4i5U64KCuIuUsO8QWcOOEPaQZe73jznaIZ3IgJmcZPL7XBdX5mwzd+YK85vnOec8j7k7IiIiFaWFXYCIiNRMCggREYlLASEiInEpIEREJC4FhIiIxKWAEBGRuBQQIoCZPW5m/5fgvqvM7Mxk1yQSNgWEiIjEpYAQSSFmlhF2DZI6FBBSawRdO6PMbJ6Z7TGzv5tZKzN7w8x2mdkUM2sas/+3zGyhmRWZ2btm1jtm2wAzmxUc9xxQv8JrfdPM5gTHfmxmxyRY4zfMbLaZ7TSztWb2uwrbTwqeryjYfnWwPtvM7jGz1Wa2w8w+DNadamaFcd6HM4PHvzOzF83sKTPbCVxtZoPMbFrwGhvM7AEzy4o5/igzm2xm28xsk5ndZmatzWyvmTWL2W+gmW0xs8xEfndJPQoIqW2+DQwFegDDgTeA24DmRP+efwJgZj2AZ4CfAi2A14F/mVlW8GH5T+AfwBHAC8HzEhx7LDAOuB5oBjwKvGJm9RKobw9wFZALfAP4gZmdFzxvh6De0UFN/YE5wXF3AwOBE4KafgFEEnxPzgVeDF5zPFAO3BS8J0OAM4AfBjXkAFOASUAboBvwtrtvBN4FvhPzvFcAz7p7aYJ1SIpRQEhtM9rdN7n7OuADYLq7z3b3/cBEYECw38XAa+4+OfiAuxvIJvoBfDyQCfzV3Uvd/UVgRsxrXAc86u7T3b3c3Z8A9gfHfSl3f9fd57t7xN3nEQ2pU4LNlwNT3P2Z4HU/c/c5ZpYGXAPc6O7rgtf8OPidEjHN3f8ZvOY+d5/p7p+4e5m7ryIacJ/X8E1go7vf4+7F7r7L3acH254gGgqYWTpwKdEQlTpKASG1zaaYx/viLDcKHrcBVn++wd0jwFqgbbBtnR84UuXqmMcdgZuDLpoiMysC2gfHfSkzG2xm7wRdMzuAkUS/yRM8x6dxDmtOtIsr3rZErK1QQw8ze9XMNgbdTn9MoAaAl4E+ZtaFaCtth7v/+zBrkhSggJBUtZ7oBz0AZmZEPxzXARuAtsG6z3WIebwW+IO758b8a+DuzyTwuk8DrwDt3b0J8Ajw+eusBbrGOWYrUFzJtj1Ag5jfI51o91SsikMyPwwsAbq7e2OiXXAHqwF3LwaeJ9rSuRK1Huo8BYSkqueBb5jZGcFJ1puJdhN9DEwDyoCfmFmGmV0ADIo59m/AyKA1YGbWMDj5nJPA6+YA29y92MwGAZfFbBsPnGlm3wlet5mZ9Q9aN+OAe82sjZmlm9mQ4JzHMqB+8PqZwK+Bg50LyQF2ArvNrBfwg5htrwKtzeynZlbPzHLMbHDM9ieBq4FvAU8l8PtKClNASEpy96VE+9NHE/2GPhwY7u4l7l4CXED0g3A70fMVE2KOzSd6HuKBYHtBsG8ifgj83sx2Af9DNKg+f941wDlEw2ob0RPU/YLNPwfmEz0Xsg24E0hz9x3Bc44l2vrZAxxwVVMcPycaTLuIht1zMTXsItp9NBzYCCwHTovZ/hHRk+OzgvMXUoeZJgwSkVhmNhV42t3Hhl2LhEsBISJfMLPjgMlEz6HsCrseCZe6mEQEADN7gug9Ej9VOAioBSEiIpVQC0JEROJKqYG9mjdv7p06dQq7DBGRWmPmzJlb3b3ivTVAigVEp06dyM/PD7sMEZFaw8xWV7ZNXUwiIhKXAkJEROJSQIiISFwKCBERiUsBISIicSkgREQkLgWEiIjEpYAQEanF8ldt49H3Dncywi+XUjfKiYjUFcWl5dw7eRl/+2AF7Zs24MohHWmQVbUf6QoIEZFaZu7aIm5+YS4Fm3dz2eAO3HZO7yoPB1BAiIjUGiVlEe5/ezkPv/cpLXPq8eQ1gzi5R9xhlKqEAkJEpBZYtH4nN78wl8UbdnLhwHb85pt9aJKdmdTXVECIiNRgZeURHn73U+6fupwm2VmMvSqPM/u0qpbXVkCIiNRQyzft4uYX5jKvcAfD+7Xh9986iqYNs6rt9RUQIiI1THnEGfvBCu6ZvIxG9TJ48LJj+cYxR1Z7HQoIEZEaZOXWPfz8hbnMXL2dr/dpxR/OP5oWOfVCqUUBISJSA0QizpPTVnHHpCVkpafxl4v7cV7/tphZaDUpIEREQrZ2215+8eI8pq34jFN7tuCOC46hdZP6YZelgBARCYu788y/1/KH1xZhZtz57aP5Tl77UFsNsRQQIiIh2LBjH7e8NJ/3l23hhK7N+POFx9CuaYOwyzqAAkJEpBq5OxNmreN3/1pIWbnz+3OP4orBHUlLqxmthlgKCBGRarJ5VzG3TVjAlMWbOK5TU+66sB+dmjcMu6xKKSBERKrBv+au5zcvL2BvSTm//kZvvndiZ9JrYKshlgJCRCSJtu0p4TcvL+C1eRvo164J93ynH91a5oRdVkIUECIiSfLWwo3cNnE+O/aVMuqsnlx/chcy0mvPPG0KCBGRKrZjbyn/+6+FTJi9jj5HNuYf3x9M7yMbh13WIVNAiIhUoXeXbuaWl+axdXcJPzm9Gzec3p2sjNrTaoilgBARqQK7ikv5w2uLeXbGWrq3bMTfrsrjmHa5YZf1lSQ1IMxsGHAfkA6Mdfc7KmxvAjwFdAhqudvdHwu2rQJ2AeVAmbvnJbNWEZHD9XHBVka9OI/1O/Zx/SlduOnMHtTPTA+7rK8saQFhZunAg8BQoBCYYWavuPuimN1+BCxy9+Fm1gJYambj3b0k2H6au29NVo0iIl/F3pIy7nxjCU9MW03n5g15ceQQBnY8IuyyqkwyWxCDgAJ3XwFgZs8C5wKxAeFAjkUHHmkEbAPKkliTiEiVWLBuBzc8PYtVn+3l6hM6ccuwXmRn1f5WQ6xkBkRbYG3MciEwuMI+DwCvAOuBHOBid48E2xx4y8wceNTdx8R7ETMbAYwA6NChQ9VVLyJSiUkLNnLTc3No2iCTp68bzAldm4ddUlIkMyDi3SLoFZbPAuYApwNdgclm9oG77wROdPf1ZtYyWL/E3d//ryeMBscYgLy8vIrPLyJSZdydR95bwZ2TltC/fS5/uyovtMl8qkMyr70qBNrHLLcj2lKI9T1ggkcVACuBXgDuvj74uRmYSLTLSkQkFCVlEX7x4jzunLSE4f3a8OyI41M6HCC5ATED6G5mnc0sC7iEaHdSrDXAGQBm1groCawws4ZmlhOsbwh8HViQxFpFRCq1bU8JV/x9Oi/MLOTGM7pz/yX9U+IqpYNJWheTu5eZ2Q3Am0Qvcx3n7gvNbGSw/RHgduBxM5tPtEvqFnffamZdgInBpBkZwNPuPilZtYqIVKZg826+/8QMNuwo5r5L+nNu/7Zhl1RtzD11uu3z8vI8Pz8/7DJEJEV8VLCVHzw1k6yMNB69Mo+BHZuGXVKVM7OZld1npjupRUTieHr6Gn7z8gK6tWjE2O/m0f6ImjXbW3VQQIiIxCiPOH98fTF//3Alp/ZswehLB5BTPzPsskKhgBARCezeX8ZPnpnN1CWbufqETvz6G71r1fDcVU0BISICrCvax/cfn8Hyzbu5/by+XHl8x7BLCp0CQkTqvNlrtnPdkzPZX1rOY1cfx8k9WoRdUo2ggBCROu3Veeu5+fm5tGxcj2euG0z3VrVjOtDqoIAQkTrJ3Rk9tYB7Jy/juE5NeeSKgTRrlNp3Rh8qBYSI1DnFpeXc8tI8Xp6znguObcufLjiaehmpf2f0oVJAiEidsnX3fkY8mc+sNUWMOqsnPzy1K8GoDVKBAkJE6oylG3fx/SdmsHX3fh66/FjOOfrIsEuq0RQQIlInvLt0Mzc8PZvsrHSeGzGEfu1r93zR1UEBISIp7/GPVvL7VxfRq3Vjxn43jza52WGXVCsoIEQkZZWVR/j9q4t4ctpqzuzdivsu6U/DevrYS5TeKRFJSTuLS/nR+Fl8sHwr15/chV8M60V6mk5GHwoFhIiknDWf7eX7T8xg5dY93Pnto7n4OM1XfzgUECKSUmas2sb1/5hJecR58vuDOKFr87BLqrUUECKSMibMKuTWl+bTtmk2464+js7NG4ZdUq2mgBCRWi8Sce6dvIwH3ilgSJdmPHzFseQ2yAq7rFpPASEitdq+knJ+/sJcXpu/gYvz2nP7eX3Jyqi7czhUJQWEiNRam3cWc92T+cxbt4NfndOba7/WWcNmVCEFhIjUSgvX7+DaJ/LZsa+UMVfmMbRPq7BLSjkKCBGpdSYv2sSNz86mSXYmL4wcwlFtmoRdUkpSQIhIrRGJOA+9W8A9k5dxdNsmjL0qj5aN64ddVspSQIhIrbCzuJSfPTeXKYs3cW7/NtxxwTFkZ2kOh2RSQIhIjbd04y5GPjWTtdv28rvhffjuCZ10MroaKCBEpEZ7Ze56bnlxHo3qZ/DMiOM5rtMRYZdUZyggRKRGKi2P8KfXlzDuo5Uc16kpD152rM43VLOk3k1iZsPMbKmZFZjZrXG2NzGzf5nZXDNbaGbfS/RYEUldm3cVc/nY6Yz7aCVXn9CJp687XuEQgqS1IMwsHXgQGAoUAjPM7BV3XxSz24+ARe4+3MxaAEvNbDxQnsCxIpKCZq7exg/Hz2LHvlL+enF/zhvQNuyS6qxktiAGAQXuvsLdS4BngXMr7ONAjkXPNjUCtgFlCR4rIinE3Xly2iouGfMJ9TPTmfjDExUOIUvmOYi2wNqY5UJgcIV9HgBeAdYDOcDF7h4xs0SOBcDMRgAjADp00JjvIrXRvpJyfjVxPhNmr+OMXi259+L+NMnODLusOi+ZARHvGjSvsHwWMAc4HegKTDazDxI8NrrSfQwwBiAvLy/uPiJSc635bC/XPzWTJRt3ctOZPfjx6d1I08xvNUIyA6IQaB+z3I5oSyHW94A73N2BAjNbCfRK8FgRqeXeWbKZG5+djZkx7urjOK1ny7BLkhjJPAcxA+huZp3NLAu4hGh3Uqw1wBkAZtYK6AmsSPBYEamlIhHnr1OWcc0TM2jXtAH/uuEkhUMNlLQWhLuXmdkNwJtAOjDO3Rea2chg+yPA7cDjZjafaLfSLe6+FSDescmqVUSqz469pdz0/BymLtnMBce25Q/nHa0hM2ooi/bupIa8vDzPz88PuwwRqcTiDTsZ+dRM1hft43++2Ycrju+oITNCZmYz3T0v3jbdSS0i1eKfs9dx64R5NMnO5NkRQxjYsWnYJclBKCBEJKlKyiL88fXFPP7xKgZ1PoIHLhtAyxzdFV0bKCBEJGk27yzmh+Nnkb96O9ee1Jlbzu5FZrrmi64tFBAikhQzVkWHzNizv4zRlw5geL82YZckh0gBISJVyt15/ONV/OG1xbQ/ogHjrx1Mj1Y5YZclh0EBISJVZm9JGb+cMJ+X56xnaJ9W3POdfjSuryEzaisFhIhUiVVb9zDyqZks3bSLUWf15AendNWQGbWcAkJEvrK3F2/ip8/NIT3NeOJ7gzi5R4uwS5IqoIAQkcNWHnHum7KM+6cW0LdtYx6+fCDtj2gQdllSRRQQInJYivaWcOOzc3hv2RYuGtiO28/rS/1MDZmRShQQInLIFqzbwQ/Gz2TjjmL+cH5fLhvUQUNmpCAFhIgckpdmFnLbxPkc0TCL568fwoAOGjIjVSkgRCQhxaXl/N9ri3jqkzUM6dKM0ZcNoHmjemGXJUmkgBCRg5q1Zjs/f34uK7bu4fqTuzDqrJ5kaMiMlJdQQJjZS8A44A13jyS3JBGpKYpLy/nLlGX87f0VHNkkm6evHcwJ3ZqHXZZUk0RbEA8TnR70fjN7AXjc3ZckrywRCdu8wiJufn4uyzfv5tJBHbjtnF7k6K7oOiWhgHD3KcAUM2sCXApMNrO1wN+Ap9y9NIk1ikg1KimLMHrqch5691NaNKrHE9cM4hTd+FYnJXwOwsyaAVcAVwKzgfHAScB3gVOTUZyIVK+F63dw8/NzWbJxFxcObMdvvtmHJtlqNdRViZ6DmAD0Av4BDHf3DcGm58xMc3yK1HKl5REeeudTRk9dTtOGWYy9Ko8z+7QKuywJWaItiAfcfWq8DZXNZSoitcPSjbu4+YU5LFi3k3P7t+F/v3UUuQ2ywi5LaoBEA6K3mc1y9yIAM2sKXOruDyWvNBFJprLyCI++v4L7piwnp34Gj1xxLMP6Hhl2WVKDJBoQ17n7g58vuPt2M7sOUECI1EIFm3dx8wvzmLu2iHOObs3t5/almW56kwoSDYg0MzN3dwAzSwfUBhWpZcojzrgPV3LXW0tpkJWuqUDlSyUaEG8Cz5vZI4ADI4FJSatKRKrcyq17GPXCXPJXb2don1b84fy+tMypH3ZZUoMlGhC3ANcDPwAMeAsYm6yiRKTqRCLOk9NWccekJWSlp/GXi/txXv+2Gn1VDirRG+UiRO+mfji55YhIVVq7bS+jXpzLJyu2cWrPFtxxwTG0bqJWgyQm0fsgugN/AvoAX/x1uXuXgxw3DLgPSAfGuvsdFbaPAi6PqaU30MLdt5nZKmAXUA6U6XJakcS5O+Onr+GPry8mzYw/f/sYLsprp1aDHJJEu5geA34L/AU4jei4TF/6lxacyH4QGAoUAjPM7BV3X/T5Pu5+F3BXsP9w4CZ33xbzNKe5+9YEaxQRYF3RPm55cR4fFmzlpG7NufPCY2ibmx12WVILJRoQ2e7+dnAl02rgd2b2AdHQqMwgoMDdVwCY2bPAucCiSva/FHgmwXpEpAJ354X8Qm5/dRHl7prpTb6yRAOi2MzSgOVmdgOwDmh5kGPaAmtjlguBwfF2NLMGwDDghpjVDrxlZg486u5jEqxVpM7ZuKOYX06YxztLt3B8lyO468J+tD+iQdhlSS2XaED8FGgA/AS4nWg303cPcky8ry1eyb7DgY8qdC+d6O7rzawl0dFjl7j7+//1ImYjgBEAHTp0OEhJIqnF3Zk4ex2/e2UhJeURfje8D1cN6URamloN8tUdNCCCcwnfcfdRwG6i5x8SUQi0j1luB6yvZN9LqNC95O7rg5+bzWwi0S6r/wqIoGUxBiAvL6+yABJJOZt3FfOriQuYvGgTeR2bcvdF/ejUvGHYZUkKOWhAuHu5mQ2MvZM6QTOA7mbWmWiX1CXAZRV3CuaYOIXoUOKfr2sIpLn7ruDx14HfH8Jri6Qsd+df8zbwPy8vYG9JOb86pzfXnNSZdLUapIol2sU0G3g5mE1uz+cr3X1CZQe4e1lwvuJNope5jnP3hWY2Mtj+SLDr+cBb7r4n5vBWwMTg5FoG8LS7685tqfM+272f37y8gNfnb6Rf+1zuuagf3Vo2CrssSVGWSKPAzB6Ls9rd/ZqqL+nw5eXleX6+pqeQ1DRpwQZ+NXEBu4rL+OnQ7oz4Whcy0tPCLktqOTObWdl9ZoneSZ3oeQcRqWLLNu3iL5OX8caCjfRt25inL+pPz9Y5YZcldUCid1I/RpwrkGpaC0IklSxav5MH3lnOGws2kp2Zzs+G9uAHp3YlU60GqSaJnoN4NeZxfaLnDSq7IklEvoJ5hUXc/3YBUxZvIqdeBj86tRvXnNSZIxpqhH2pXol2Mb0Uu2xmzwBTklKRSB01c/V2Rk9dzrtLt9C4fgY3ndmDq0/sRJPszLBLkzoq0RZERd0B3ZUmUgWmr/iM0VML+LBgK00bZDLqrJ5cNaQjOfUVDBKuRM9B7OLAcxAbic4RISKHwd35+NPPuP/t5UxfuY3mjepx2zm9uHxwRxrWO9zvbSJVK9EuJl0yIVIF3J33lm1h9NQCZq7eTqvG9fjt8D5cOqgD9TPTwy5P5ACJtiDOB6a6+45gORc41d3/mcziRFKFu/P24s2MnrqcuYU7aNOkPref15eLBrZTMEiNlWhb9rfuPvHzBXcvMrPfAgoIkS8RiThvLtzI6KkFLNqwk/ZHZHPHBUdzwbHtyMrQ5apSsyUaEPH+ktVRKlKJ8ojz2vwNPDB1Ocs27aZz84bcfVE/zu3fRvcxSK2R6Id8vpndS3SGOAd+DMxMWlUitVRZeYRX5q7ngXcKWLFlD91bNuK+S/rzzWPaaDA9qXUSDYgfA78BnguW3wJ+nZSKRGqh0vIIE2et48F3C1j92V56tc7hocuPZdhRrTU3g9RaiV7FtAe4Ncm1iNQ6+8vKeSG/kIff/ZR1Rfs4um0Txlw5kDN7t1IwSK2X6FVMk4GL3L0oWG4KPOvuZyWzOJGaqri0nGf/vYZH3lvBxp3FDOiQy/+d15dTe7bQHNCSMhLtYmr+eTgAuPv2YCpQkTplb0kZT09fw6Pvr2DLrv0M6nQEd1/UjxO7NVMwSMpJNCAiZtbB3dcAmFknKp9fWiTl7N5fxj+mrWbsByv4bE8JJ3RtxuhLB3B8l2ZhlyaSNIkGxK+AD83svWD5ZGBEckoSqRlKyyNs3FHMP2ev4+8fraRobymn9GjBT87oxsCOR4RdnkjSJXqSepKZ5RENhTnAy8C+ZBYmkmy7iktZX1TMuqK9rCsqZt32fawv2se6oujPTTuLiQTt5DN7t+SG07vTv31uuEWLVKNET1JfC9wItCMaEMcD04DTk1eayOGLRJwtu/dTGHzox37wf75uZ3HZAcdkphtHNsmmTW59hnRtRrvcbNrkZtO/Qy69WjcO6TcRCU+iXUw3AscBn7j7aWbWC/jf5JUl8uWKS8sP+NBft31ftBVQtJf1RcVs2LGP0vIDT5Pl1M+gbW42bXOzGdT5CNoEj9vkZtOuaTbNG9XTzWwiMRINiGJ3LzYzzKyeuy8xs55JrUyq3dpte3n841W8tWgj7pCVnkZGupGZnkZGehpZ6UZGWhqZGWlkpn2+3shKT/vicWZ6GpkVjwm2f7E+zcjKSPvicfT5otujx0Sfq7Q8wvqi4gODIPi5dXfJAbWnGbRqXD/6jb99LuccfSRtm2bTNrc+bXMbcGRufRprfgWRQ5JoQBQGI7j+E5hsZtvRlKMpwd3598ptjPtoJZMXbSLNjNN6tSSnXgalEae0LEJpeeSLx2WRCHv3lVNWHl1fVu6UVHhcVu7R5UjVXehWPzMt+u2/aQOOatP4i2/+n7cCWjeprzGORKpYoiepzw8e/s7M3gGaAJOSVpUkXUlZhFfnrWfcRytZsG4nuQ0yGXlKV64c0pEjm2RXyWu4O6XlTlkkQmlZEB7B49JI/FCJfZyeZl8EQdMGmbrPQKSaHfKIrO7+3sH3kprqs937GT99Df/4ZDVbdu2nW8tG/PH8ozl/QFuys6p2XgIzIyvDyCINsqr0qUWkGmjI7jpiycadPPbhKibOWUdJWYRTerTgmos6c3L35vpmLiJxKSBSWCTivLtsM3//cCUfFXxG/cw0LhrYju+d2IluLTWLrIh8OQVECtqzv4yXZhXy2EerWLl1D60b1+cXw3py6XEdaNpQfT0ikpikBoSZDQPuA9KBse5+R4Xto4DLY2rpDbRw920HO1b+27qifTz58Sqe+fcadhaX0a99LvdfOoCz+7bWFT4icsiSFhBmlk50BrqhQCEww8xecfdFn+/j7ncBdwX7DwduCsLhoMdKlLsza812xn24ikkLNwIwrG9rrjmxMwM7Ng25OhGpzZLZghgEFLj7CgAzexY4F6jsQ/5S4JnDPLbOKS2P8Pr8DYz7aBVz1xbRuH4G136tM1cN6UTb3Kq5TFVE6rZkBkRbYG3MciEwON6OZtYAGAbccBjHjiAYWbZDhw5freJaYPueEp7+9xr+MW01G3cW06V5Q24/9yi+PbAdDbJ0SklEqk4yP1HiXTtZ2a21w4GP3H3boR7r7mOAMQB5eXkpO0dFweZdjPtoFRNmFVJcGuGkbs350wVHc0qPFpraUkSSIpkBUQi0j1luR+XDc1zCf7qXDvXYlOXuvLdsC+M+WsX7y7aQlZHGBQPa8r0TO9OztS5TFZHkSmZAzAC6m1lnYB3RELis4k5m1gQ4BbjiUI9NVftKypkwO3qZasHm3bTIqcfNQ3tw2eAONGtUL+zyRKSOSFpAuHuZmd0AvEn0UtVx7r7QzEYG2x8Jdj0feMvd9xzs2GTVWlNs3FHME9Oil6kW7S2lb9vG/OXifnzj6DZkZegyVRGpXuaeOt32eXl5np+fH3YZh6Vg826Gj/6Q/WXlfL1Pa645qTPHdWqqYTBEJKnMbKa758Xbpsteaog7Jy0hPc14++ZT6dy8YdjliIigfosaIH/VNiYv2sTIU7ooHESkxlBAhMzd+ePri2mZU49rTuocdjkiIl9QQITszYWbmLWmiJuG9tCNbiJSoyggQlRaHuHPk5bQrWUjLhrYLuxyREQOoIAI0XMz1rJi6x5uGdaLDI22KiI1jD6VQrJnfxl/nbKc4zo15czeLcMuR0TkvyggQjL2g5Vs3b2fW8/urXsdRKRGUkCEYOvu/Yx5/1OGHdVaczaISI2lgAjB/W8vp7gswqhhPcMuRUSkUgqIarZy6x6enr6GS45rT9cWjcIuR0SkUgqIanb3m0vJykjjxjO7h12KiMiXUkBUozlri3ht/gau+1oXWubUD7scEZEvpYCoJu7On15fTPNGWVx3cpewyxEROSgFRDWZumQz01du48YzutOonobUEJGaTwFRDcojzp2TltC5eUMuGdQh7HJERBKigKgGL80sZNmm3Yw6qyeZGlJDRGoJfVol2b6Scu6dvIz+7XM5u2/rsMsREUmYAiLJHvt4JRt3FvPLs3tpSA0RqVUUEEm0fU8JD7/7KWf0asngLs3CLkdE5JAoIJLogXcK2LO/jFvO7hV2KSIih0wBkSRrt+3lH9NWc9HA9vRolRN2OSIih0wBkST3vLWUtDS4aWiPsEsRETksCogkWLBuB/+cs55rTuxM6yYaUkNEaicFRBLc8cYSmjbIZOSpXcMuRUTksCkgqtj7y7bwYcFWbji9O43rZ4ZdjojIYVNAVKFIxLnjjSW0a5rNFcdrSA0Rqd2SGhBmNszMlppZgZEVSQgAAAoISURBVJndWsk+p5rZHDNbaGbvxaxfZWbzg235yayzqrw8dx2LNuxk1Fk9qZeRHnY5IiJfSdKGFTWzdOBBYChQCMwws1fcfVHMPrnAQ8Awd19jZi0rPM1p7r41WTVWpeLScu5+cxl92zZm+DFtwi5HROQrS2YLYhBQ4O4r3L0EeBY4t8I+lwET3H0NgLtvTmI9SfXUJ6tZV7SPX57dm7Q0DakhIrVfMgOiLbA2ZrkwWBerB9DUzN41s5lmdlXMNgfeCtaPqOxFzGyEmeWbWf6WLVuqrPhDsWNvKaOnFnByjxac2K15KDWIiFS1ZM5cE+9rtMd5/YHAGUA2MM3MPnH3ZcCJ7r4+6HaabGZL3P39/3pC9zHAGIC8vLyKz18tHnqvgJ3Fpdw6TENqiEjqSGYLohBoH7PcDlgfZ59J7r4nONfwPtAPwN3XBz83AxOJdlnVOOuL9vHYR6s4v39b+rRpHHY5IiJVJpkBMQPobmadzSwLuAR4pcI+LwNfM7MMM2sADAYWm1lDM8sBMLOGwNeBBUms9bDdO3kZOPzs6xpSQ0RSS9K6mNy9zMxuAN4E0oFx7r7QzEYG2x9x98VmNgmYB0SAse6+wMy6ABOD+RMygKfdfVKyaj1cSzbu5KVZhVx7UmfaNW0QdjkiIlUqmecgcPfXgdcrrHukwvJdwF0V1q0g6Gqqye58Ywk59TL40Wndwi5FRKTK6U7qwzTt0894Z+kWfnRaN3IbZIVdjohIlVNAHAZ35443FtOmSX2+e0KnsMsREUkKBcRheG3+BuYW7uBnX+9J/UwNqSEiqUkBcYhKyiLc9eZSerXO4fwBFe/7ExFJHQqIQ/T09NWs/mwvt5zdi3QNqSEiKUwBcQh2FZdy/9QChnRpxqk9WoRdjohIUikgDsGY91ewbU8JvzynF8E9GiIiKUsBkaDNO4sZ+8FKvnnMkRzTLjfsckREkk4BkaC/TFlOWSTCqLN6hl2KiEi1UEAkoGDzbp7PX8vlgzvSsVnDsMsREakWCogE/HnSErIz0/nx6RpSQ0TqDgXEQeSv2sZbizYx8pQuNGtUL+xyRESqjQLiS7g7f3pjCS1z6nHNSZ3DLkdEpFopIL7Emws3MXP1dm4a2oMGWUkd+FZEpMZRQFSirDzCn99cQtcWDbloYLuwyxERqXYKiEo8l7+WFVv2cMuwXmSk620SkbpHn3xx7C0p469TlnNcp6YM7dMq7HJEREKhgIhj7Acr2bJrP7ee3VtDaohInaWAqGDr7v08+t6nDDuqNQM7Ng27HBGR0CggKhj99nKKyyKMGqYhNUSkblNAxFi1dQ/jp6/hkuPa07VFo7DLEREJlQIixl1vLSUrI40bz+wedikiIqFTQATmrC3itXkbuPZrXWiZUz/sckREQqeAIBhS4/XFNG+UxYiTu4RdjohIjaCAAN5ZupnpK7fxkzO606iehtQQEQEFBOUR5843ltK5eUMuHdQh7HJERGqMOv91eV9pOf3b53JarxZkakgNEZEvJPUT0cyGmdlSMysws1sr2edUM5tjZgvN7L1DObYqNKqXwZ0XHsOwvkcm6yVERGqlpLUgzCwdeBAYChQCM8zsFXdfFLNPLvAQMMzd15hZy0SPFRGR5EpmC2IQUODuK9y9BHgWOLfCPpcBE9x9DYC7bz6EY0VEJImSGRBtgbUxy4XBulg9gKZm9q6ZzTSzqw7hWADMbISZ5ZtZ/pYtW6qodBERSeZJ6njDoHqc1x8InAFkA9PM7JMEj42udB8DjAHIy8uLu4+IiBy6ZAZEIdA+ZrkdsD7OPlvdfQ+wx8zeB/oleKyIiCRRMruYZgDdzayzmWUBlwCvVNjnZeBrZpZhZg2AwcDiBI8VEZEkSloLwt3LzOwG4E0gHRjn7gvNbGSw/RF3X2xmk4B5QAQY6+4LAOIdm6xaRUTkv5l76nTb5+XleX5+fthliIjUGmY2093z4m5LpYAwsy3A6sM8vDmwtQrLqc30XhxI78eB9H78Ryq8Fx3dvUW8DSkVEF+FmeVXlqJ1jd6LA+n9OJDej/9I9fdCgw+JiEhcCggREYlLAfEfY8IuoAbRe3EgvR8H0vvxHyn9XugchIiIxKUWhIiIxKWAEBGRuOp8QFTXxES1gZm1N7N3zGxxMIHTjWHXFDYzSzez2Wb2ati1hM3Mcs3sRTNbEvyNDAm7pjCZ2U3B/5MFZvaMmdUPu6aqVqcDImZiorOBPsClZtYn3KpCVQbc7O69geOBH9Xx9wPgRqLjgwncB0xy915EB9Wss++LmbUFfgLkuXtfokMCXRJuVVWvTgcEmpjoAO6+wd1nBY93Ef0AiDsPR11gZu2AbwBjw64lbGbWGDgZ+DuAu5e4e1G4VYUuA8g2swygASk44nRdD4iEJyaqa8ysEzAAmB5uJaH6K/ALogNJ1nVdgC3AY0GX21gzaxh2UWFx93XA3cAaYAOww93fCreqqlfXAyLhiYnqEjNrBLwE/NTdd4ZdTxjM7JvAZnefGXYtNUQGcCzwsLsPAPYAdfacnZk1Jdrb0BloAzQ0syvCrarq1fWA0MREFZhZJtFwGO/uE8KuJ0QnAt8ys1VEux5PN7Onwi0pVIVAobt/3qJ8kWhg1FVnAivdfYu7lwITgBNCrqnK1fWA0MREMczMiPYxL3b3e8OuJ0zu/kt3b+funYj+XUx195T7hpgod98IrDWznsGqM4BFIZYUtjXA8WbWIPh/cwYpeNI+mVOO1niVTWoUcllhOhG4EphvZnOCdbe5++sh1iQ1x4+B8cGXqRXA90KuJzTuPt3MXgRmEb36bzYpOOyGhtoQEZG46noXk4iIVEIBISIicSkgREQkLgWEiIjEpYAQEZG4FBAiNYCZnaoRY6WmUUCIiEhcCgiRQ2BmV5jZv81sjpk9GswXsdvM7jGzWWb2tpm1CPbtb2afmNk8M5sYjN+DmXUzsylmNjc4pmvw9I1i5lsYH9yhKxIaBYRIgsysN3AxcKK79wfKgcuBhsAsdz8WeA/4bXDIk8At7n4MMD9m/XjgQXfvR3T8ng3B+gHAT4nOTdKF6J3tIqGp00NtiByiM4CBwIzgy302sJnocODPBfs8BUwwsyZArru/F6x/AnjBzHKAtu4+EcDdiwGC5/u3uxcGy3OATsCHyf+1ROJTQIgkzoAn3P2XB6w0+02F/b5s/Jov6zbaH/O4HP3/lJCpi0kkcW8DF5pZSwAzO8LMOhL9f3RhsM9lwIfuvgPYbmZfC9ZfCbwXzK9RaGbnBc9Rz8waVOtvIZIgfUMRSZC7LzKzXwNvmVkaUAr8iOjkOUeZ2UxgB9HzFADfBR4JAiB29NMrgUfN7PfBc1xUjb+GSMI0mqvIV2Rmu929Udh1iFQ1dTGJiEhcakGIiEhcakGIiEhcCggREYlLASEiInEpIEREJC4FhIiIxPX/ZZJPRY5ua5sAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de3xdZZ3v8c9v7+zcb72E3pJe4CCXliYtpRZRBgQZ7qggVAEFlQ7IjOJxHC8zinqOZzxnHAYRBauAogzIcBsGQREFBQeQtrSlUG5CS9NrWmiaNPfkd/5YK+nO7k6apFnZSfb3/Xrt1957rWet/Uug+5u1nrWex9wdERHJXrFMFyAiIpmlIBARyXIKAhGRLKcgEBHJcgoCEZEspyAQEclyCgKRATKzn5rZ/x5g2w1mdurB7kdkJCgIRESynIJARCTLKQhkXAlPyXzRzNaa2V4zu8XMppjZI2bWYGaPmdmEpPbnmtmLZrbbzJ4ws6OS1i0ws1Xhdr8E8lM+62wzWx1u+99mNn+INV9hZq+b2dtm9qCZTQ+Xm5n9m5ntMLP68GeaF64708xeCmvbbGZ/P6RfmAgKAhmfzgc+ALwLOAd4BPgqMJng//nPApjZu4A7gWuACuBh4L/MLNfMcoEHgJ8DE4H/CPdLuO1C4Fbgb4BJwI+AB80sbzCFmtn7gX8GLgSmARuBu8LVpwEnhj9HOXARsCtcdwvwN+5eAswDfj+YzxVJpiCQ8ej77r7d3TcDTwLPuvvz7t4K3A8sCNtdBPzK3X/r7u3Ad4EC4D3AEiABXO/u7e5+D/Bc0mdcAfzI3Z919053/xnQGm43GBcDt7r7qrC+rwDHm9lsoB0oAY4EzN3Xu/vWcLt24GgzK3X3d9x91SA/V6SHgkDGo+1Jr5vTvC8OX08n+AscAHfvAjYBM8J1m733qIwbk17PAr4QnhbabWa7gapwu8FIraGR4K/+Ge7+e+BG4AfAdjNbbmalYdPzgTOBjWb2BzM7fpCfK9JDQSDZbAvBFzoQnJMn+DLfDGwFZoTLus1Mer0J+La7lyc9Ct39zoOsoYjgVNNmAHe/wd2PBeYSnCL6Yrj8OXc/DziE4BTW3YP8XJEeCgLJZncDZ5nZKWaWAL5AcHrnv4GngQ7gs2aWY2YfBhYnbftj4Eoze3fYqVtkZmeZWckga/h34HIzqwn7F/4PwamsDWZ2XLj/BLAXaAE6wz6Mi82sLDyltQfoPIjfg2Q5BYFkLXd/BbgE+D6wk6Bj+Rx3b3P3NuDDwGXAOwT9CfclbbuCoJ/gxnD962HbwdbwO+BrwL0ERyGHAUvD1aUEgfMOwemjXQT9GACXAhvMbA9wZfhziAyJaWIaEZHspiMCEZEspyAQEclyCgIRkSynIBARyXI5mS5gsCZPnuyzZ8/OdBkiImPKypUrd7p7Rbp1Yy4IZs+ezYoVKzJdhojImGJmG/tap1NDIiJZTkEgIpLlFAQiIlluzPURpNPe3k5tbS0tLS2ZLmXcyM/Pp7KykkQikelSRCRi4yIIamtrKSkpYfbs2fQeLFKGwt3ZtWsXtbW1zJkzJ9PliEjExsWpoZaWFiZNmqQQGCZmxqRJk3SEJZIlxkUQAAqBYabfp0j2iDwIzCxuZs+b2UNp1p0UTsq9Onx8Pao6Wto72VrfTGeXRlsVEUk2EkcEnwPW97P+SXevCR/fiqqIto4u6hpaaWkf/vk7du/ezQ9/+MNBb3fmmWeye/fuYa9HRGQwIg0CM6sEzgJ+EuXnDERBbhyApraRC4LOzv4/6+GHH6a8vHzY6xERGYyojwiuB/4B6OqnzfFmtsbMHjGzuekamNkyM1thZivq6uqGVEgiHiM3HqOprWNI2/fny1/+Mn/5y1+oqanhuOOO4+STT+ZjH/sYxxxzDAAf/OAHOfbYY5k7dy7Lly/v2W727Nns3LmTDRs2cNRRR3HFFVcwd+5cTjvtNJqbm4e9ThGRdCK7fNTMzgZ2uPtKMzupj2argFnu3mhmZxJMwn14aiN3Xw4sB1i0aFG/J/m/+V8v8tKWPWnXtXZ00tkFheHRwUAdPb2Ua89Jm1EAfOc732HdunWsXr2aJ554grPOOot169b1XHp56623MnHiRJqbmznuuOM4//zzmTRpUq99vPbaa9x55538+Mc/5sILL+Tee+/lkks0+6CIRC/KI4ITgHPNbANwF/B+M/tFcgN33+PujeHrh4GEmU2OqqCYGe5O1N3Fixcv7nX9/Q033EB1dTVLlixh06ZNvPbaa/ttM2fOHGpqagA49thj2bBhQ8RViogEIjsicPevAF+B4Oog4O/dvdefuGY2Fdju7m5miwmCadfBfG5/f7nvbe3gL3WNzJ5URGlBdHfMFhUV9bx+4okneOyxx3j66acpLCzkpJNOSnt9fl5eXs/reDyuU0MiMmJG/M5iM7sSwN1vBi4ArjKzDqAZWOrukf3BXpCIYxhNbZ3DGgQlJSU0NDSkXVdfX8+ECRMoLCzk5Zdf5plnnhm2zxURGQ4jEgTu/gTwRPj65qTlNwI3jkQNALGYkZ8Y/g7jSZMmccIJJzBv3jwKCgqYMmVKz7rTTz+dm2++mfnz53PEEUewZMmSYf1sEZGDZRH+AR6JRYsWeerENOvXr+eoo44a0Pa17zRR39zO0dNKdffsAQzm9yoio5uZrXT3RenWjZshJgaqMDeHzi6ntaO/K1pFRLJHFgZBcOlocwQ3lomIjEVZFwR5OTFiZjRFMNSEiMhYlHVBYGYU5MZpjuAOYxGRsSjrggCC00PN7V10jbGOchGRKGRtELg7LeonEBHJziAoSAS3T2Sqn6C4uBiALVu2cMEFF6Rtc9JJJ5F6mWyq66+/nqampp73GtZaRIYiK4MgETcS8VgkQ1IPxvTp07nnnnuGvH1qEGhYaxEZiqwMAjOjIBEftktIv/SlL/Waj+Ab3/gG3/zmNznllFNYuHAhxxxzDP/5n/+533YbNmxg3rx5ADQ3N7N06VLmz5/PRRdd1GusoauuuopFixYxd+5crr32WiAYyG7Lli2cfPLJnHzyycC+Ya0BrrvuOubNm8e8efO4/vrrez5Pw12LSKoRH2soco98Gba9cMBm0zu7aOvowvOC8Yf6NfUYOOM7fa5eunQp11xzDZ/5zGcAuPvuu/n1r3/N5z//eUpLS9m5cydLlizh3HPP7fNu5ptuuonCwkLWrl3L2rVrWbhwYc+6b3/720ycOJHOzk5OOeUU1q5dy2c/+1muu+46Hn/8cSZP7j1g68qVK7ntttt49tlncXfe/e5381d/9VdMmDBBw12LyH6y8ogAIB5+Hw/HHMYLFixgx44dbNmyhTVr1jBhwgSmTZvGV7/6VebPn8+pp57K5s2b2b59e5/7+OMf/9jzhTx//nzmz5/fs+7uu+9m4cKFLFiwgBdffJGXXnqp33qeeuopPvShD1FUVERxcTEf/vCHefLJJwENdy0i+xt/RwT9/OXeS1cXb2zZw5TSfKaU5h/0x15wwQXcc889bNu2jaVLl3LHHXdQV1fHypUrSSQSzJ49O+3w08nSHS28+eabfPe73+W5555jwoQJXHbZZQfcT3/jR2m4axFJlb1HBLEY+TnD10+wdOlS7rrrLu655x4uuOAC6uvrOeSQQ0gkEjz++ONs3Lix3+1PPPFE7rjjDgDWrVvH2rVrAdizZw9FRUWUlZWxfft2HnnkkZ5t+hr++sQTT+SBBx6gqamJvXv3cv/99/O+971vWH5OERl/xt8RwSAU5MZpaOnA3Q96JNK5c+fS0NDAjBkzmDZtGhdffDHnnHMOixYtoqamhiOPPLLf7a+66iouv/xy5s+fT01NDYsXLwagurqaBQsWMHfuXA499FBOOOGEnm2WLVvGGWecwbRp03j88cd7li9cuJDLLrusZx+f/vSnWbBggU4DiUhaWTcMdbJdja1s3t3MkVNLyM0Z3DzG2UDDUIuMHxkdhtrM4mb2vJk9lGadmdkNZva6ma01s4Xp9hGV7pFIM30/gYhIJo1EH8HngPV9rDsDODx8LANuGoF6euQl4sFIpAoCEclikQaBmVUCZwE/6aPJecDtHngGKDezaUP5rKGc4oqZkT+MN5aNJ2PtlKGIDF3URwTXA/8A9DUd2AxgU9L72nBZL2a2zMxWmNmKurq6/XaSn5/Prl27hvTlFYxE2qmRSJO4O7t27SI//+AvqxWR0S+yq4bM7Gxgh7uvNLOT+mqWZtl+38juvhxYDkFncer6yspKamtrSRcSB9LU1snbe9vofDuP3JysvZp2P/n5+VRWVma6DBEZAVFePnoCcK6ZnQnkA6Vm9gt3Tx7PoBaoSnpfCWwZ7AclEgnmzJkzpCLf2tXE+f/yON/+0DwufvesIe1DRGQsi+xPYHf/irtXuvtsYCnw+5QQAHgQ+Hh49dASoN7dt0ZVUzpVEwuYWJTLmk0avllEstOI31BmZlcCuPvNwMPAmcDrQBNweQbqobqyjNUKAhHJUiMSBO7+BPBE+PrmpOUOXD0SNfSnuqqcJ16to7G1g+K8rL7ZWkSykHpHgZqqctxhba2OCkQk+ygIgOrKYFavNZvqM1yJiMjIUxAAE4pymTWpkNWb3sl0KSIiI05BEKqpKtcRgYhkJQVBqLqynG17WthW3/+kLyIi442CIFQzM+gn0GWkIpJtFASho6eVkogba3TlkIhkGQVBKD8R56hppax+S0EgItlFQZCkurKcFzbX09mlkUhFJHsoCJJUV5XT2NrBG3WNmS5FRGTEKAiS1FQFHcbPq8NYRLKIgiDJoZOLKMnP0UikIpJVFARJYjGjurJcl5CKSFZREKSorirj5W0NtLRrHmMRyQ4KghQ1VRPo7HLWbdZwEyKSHRQEKaorywDdYSwi2SOyIDCzfDP7s5mtMbMXzeybadqcZGb1ZrY6fHw9qnoG6pDSfKaX5bOmVkcEIpIdopyOqxV4v7s3mlkCeMrMHnH3Z1LaPenuZ0dYx6DVzCzXkNQikjWinLze3b37zqxE+BgTt+xWV5az6e1mdjW2ZroUEZHIRdpHYGZxM1sN7AB+6+7Ppml2fHj66BEzm9vHfpaZ2QozW1FXVxdlycC+G8s0AJ2IZINIg8DdO929BqgEFpvZvJQmq4BZ7l4NfB94oI/9LHf3Re6+qKKiIsqSAZg3o4yYwWpNVCMiWWBErhpy993AE8DpKcv3dJ8+cveHgYSZTR6JmvpTlJfDu6aU6A5jEckKUV41VGFm5eHrAuBU4OWUNlPNzMLXi8N6dkVV02DUVJWzpnY37mOiW0NEZMiiPCKYBjxuZmuB5wj6CB4ysyvN7MqwzQXAOjNbA9wALPVR8s1bXVXO7qZ2Nu5qynQpIiKRiuzyUXdfCyxIs/zmpNc3AjdGVcPB6O4wXr1pN7MnF2W4GhGR6OjO4j4cfkgxBYm47jAWkXFPQdCHnHiMYyrLFAQiMu4pCPpRU1XOS1v20NbRlelSREQioyDoR3VlOW2dXby8bU+mSxERiYyCoB81M/d1GIuIjFcKgn5ML8tncnGegkBExjUFQT/MjJoqTV0pIuObguAAaqrKeKNuL/XN7ZkuRUQkEgqCA6gObyx7QRPViMg4pSA4gPmV3R3GmqhGRMYnBcEBlBUkOLSiSENSi8i4pSAYgO4O41EyHp6IyLBSEAxATVU5Oxtb2VLfkulSRESGnYJgAHpGIn1Ll5GKyPijIBiAI6eWkhuPaQ5jERmXFAQDkJsT4+jppbqxTETGpSinqsw3sz+b2Roze9HMvpmmjZnZDWb2upmtNbOFUdVzsGqqynmhtp6OTo1EKiLjS5RHBK3A+929GqgBTjezJSltzgAODx/LgJsirOeg1FSV09zeyWs7GjNdiojIsIosCDzQ/a2ZCB+p11+eB9wetn0GKDezaVHVdDCSp64UERlPIu0jMLO4ma0GdhBMXv9sSpMZwKak97XhstT9LDOzFWa2oq6uLrqC+zFrUiHlhQnWKAhEZJyJNAjcvdPda4BKYLGZzUtpYuk2S7Of5e6+yN0XVVRURFHqAZkZ1ZUaiVRExp8RuWrI3XcDTwCnp6yqBaqS3lcCW0aipqGorirn1e0N7G3tyHQpIiLDJsqrhirMrDx8XQCcCryc0uxB4OPh1UNLgHp33xpVTQerpqqMLod1mzXukIiMH1EeEUwDHjeztcBzBH0ED5nZlWZ2ZdjmYeAN4HXgx8BnIqznoFVXqsNYRMafnKh27O5rgQVplt+c9NqBq6OqYbhNKs6jamKB7jAWkXFFdxYPUk3VBI05JCLjioJgkKory9hS38KOPRqJVETGBwXBIHXfWLZGU1eKyDihIBikeTPKiMdMU1eKyLihIBik/EScI6eWsEZTV4rIOKEgGIKaqnLWbNpNV5emrhSRsU9BMATVVeU0tHbwxs69mS5FROSgKQiGYEF3h7FuLBORcUBBMASHVhRTnJejO4xFZFxQEAxBPGYcM6NMdxiLyLgwoCAws8+ZWWk4ONwtZrbKzE6LurjRrGZmOeu37qGlvTPTpYiIHJSBHhF80t33AKcBFcDlwHciq2oMqK4sp73TeWnrnkyXIiJyUAYaBN0TyJwJ3Obua0g/qUzWWDBTHcYiMj4MNAhWmtmjBEHwGzMrAbqiK2v0m1Kaz9TSfHUYi8iYN9BhqD8F1ABvuHuTmU0kOD2U1aqrynREICJj3kCPCI4HXnH33WZ2CfBPQNaPsVBTNYENu5p4Z29bpksRERmygQbBTUCTmVUD/wBsBG7vbwMzqzKzx81svZm9aGafS9PmJDOrN7PV4ePrg/4JMqi6qgxAl5GKyJg20CDoCGcTOw/4nrt/Dyg50DbAF9z9KGAJcLWZHZ2m3ZPuXhM+vjXgykeB+ZXlmGnqShEZ2wYaBA1m9hXgUuBXZhYHEv1t4O5b3X1V+LoBWA/MOJhiR5vivBwOP6RY/QQiMqYNNAguAloJ7ifYRvCF/i8D/RAzm00wf/GzaVYfb2ZrzOwRM5s70H2OFjVV5ayprSc4YBIRGXsGFAThl/8dQJmZnQ20uHu/fQTdzKwYuBe4JrwpLdkqYJa7VwPfBx7oYx/LzGyFma2oq6sbyMeOmOqqct7e28amt5szXYqIyJAMdIiJC4E/Ax8BLgSeNbMLBrBdgiAE7nD3+1LXu/sed28MXz8MJMxscpp2y919kbsvqqioGEjJI6a6MrixbLU6jEVkjBroqaF/BI5z90+4+8eBxcDX+tvAzAy4BVjv7tf10WZq2A4zWxzWs2ugxY8GR0wtIT8RY/VbCgIRGZsGekNZzN13JL3fxYFD5ASCzuUXzGx1uOyrwEwAd78ZuAC4ysw6gGZgqY+xk+2JeIx50zUSqYiMXQMNgl+b2W+AO8P3FwEP97eBuz/FAcYjcvcbgRsHWMOoVVNVzs+f2Uh7ZxeJuEb2FpGxZaCdxV8ElgPzgWpgubt/KcrCxpLqqnJaO7p4ZVtDpksRERm0gR4R4O73EnT8SoqacOrK1Zt2M29GWYarEREZnH6PCMyswcz2pHk0mJkG4g9VTihgUlGu7jAWkTGp3yMCdz/QMBICmBnVVeW6w1hExiT1bA6TmqpyXq9rpKGlPdOliIgMioJgmFRXleMOL9Rm/ejcIjLGKAiGSXVl0EmsO4xFZKxREAyT8sJc5kwu0h3GIjLmKAiGUXWl7jAWkbFHQTCMaqrK2b6nla31GolURMYOBcEwqg5vLNNlpCIyligIhtHR00tJxI3Vm3TlkIiMHQqCYZSXE+foaaWs3vROpksRERkwBcEwq6kq54Xaejq7xtRo2iKSxRQEw6y6qpy9bZ28vqMx06WIiAyIgmCYqcNYRMYaBcEwmzOpiNL8HJ5XEIjIGBFZEJhZlZk9bmbrzexFM/tcmjZmZjeY2etmttbMFkZVz0iJxTQSqYiMLVEeEXQAX3D3o4AlwNVmdnRKmzOAw8PHMuCmCOsZMTVV5byyvYHmts5MlyIickCRBYG7b3X3VeHrBmA9MCOl2XnA7R54Big3s2lR1TRSqivL6exy1m3R/QQiMvqNSB+Bmc0GFgDPpqyaAWxKel/L/mGBmS0zsxVmtqKuri6qMoeNOoxFZCyJPAjMrJhgruNr3D11ektLs8l+F+C7+3J3X+TuiyoqKqIoc1hVlOQxo7xAHcYiMiZEGgRmliAIgTvc/b40TWqBqqT3lcCWKGsaKTXqMBaRMSLKq4YMuAVY7+7X9dHsQeDj4dVDS4B6d98aVU0jqaaqnNp3mtnZ2JrpUkRE+tXv5PUH6QTgUuAFM1sdLvsqMBPA3W8GHgbOBF4HmoDLI6xnRCX3E5xy1JQMVyMi0rfIgsDdnyJ9H0ByGweujqqGTJo3o5R4zBQEIjLq6c7iiBTm5vCuKSXqMBaRUU9BEKGaqjLWbNpNcOAjIjI6ZU8QdHXBm38c0Y+sqSpnT0sHb+7cO6KfKyIyGNkTBM//HH52Djz5rzBCf6H3dBhrQnsRGcWyJwiqPwrHXAi/+xb8+ivBEULEDj+khMLcOGs0daWIjGJRXj46uuTkwod+BEUV8MwPYG8dfPCmYHlE4jHjmBll6jAWkVEte44IAGIx+Otvw6nfgHX3wJ0XQWu0M4nVzCxn/ZY9tHZoJFIRGZ2yKwgAzOC9n4dzb4Q3ngj6DfbujOzjairLaevsYv3Whsg+Q0TkYGRfEHRbeClcdAfseAlu/WvY/VYkH6ORSEVktMveIAA48ky49IGgv+CW02D7S8P+EdPK8jmkJE9BICKjVnYHAcCs4+HyR4LXt50Obz0zrLs3C6auXK0gEJFRSkEAMGUufPI3wRVFt58HrzwyrLuvqSrnjZ17qW9qH9b9iogMBwVBtwmzgjA45Ci462J4/hfDtusa3VgmIqOYgiBZ0WT4xEMw50T4z6vhqeuH5S7kYyrLMFOHsYiMTgqCVHnF8LG7Yd758Ni18Og/HfRdyKX5CQ6rKFY/gYiMStlzZ/Fg5OTCh38ChZPh6RuDq4rO+wHEE0PeZXVlOX94dQfuTjB5m4jI6BDlVJW3mtkOM1vXx/qTzKzezFaHj69HVcuQxGJwxv+F938N1v4S7lwKbUMfRbRmZjk7G9vYvLt5GIsUETl4UZ4a+ilw+gHaPOnuNeHjWxHWMjRmcOLfwzk3wF9+Dz87F5reHtKuaiqDDmOdHhKR0SayIHD3PwJD+9YcbY79BFz4c9j2Atx6OtTXDnoXR04rITcnpg5jERl1Mt1ZfLyZrTGzR8xsbl+NzGyZma0wsxV1dXUjWd8+R50Nl94HDVuDu5B3vDyozRPxGPOml+qIQERGnUwGwSpglrtXA98HHuirobsvd/dF7r6ooqJixArcz+z3wuUPQ1dHMD7Rpj8PavPqqnJe2FxPR2f0cyGIiAxUxoLA3fe4e2P4+mEgYWaTM1XPgE09Bj71KBRODPoMXn10wJvWVJXT0t7Fq9ujHfpaRGQwMhYEZjbVwusozWxxWMuuTNUzKBNmwycfhYp3BVcTrb5zQJt132Gs00MiMppEefnoncDTwBFmVmtmnzKzK83syrDJBcA6M1sD3AAsdR+hyYSHQ3FFcBfy7BPggSvhTzcccJOZEwuZUJhQh7GIjCqR3VDm7h89wPobgRuj+vwRkV8KF98D9y2D334tuPHsA98KLjtNo3sk0uc3vTPChYqI9C3TVw2NfTl5cMGtcNwV8N83wAOfgc6+Rxk94bDJvLq9kf/10Et0do2dAyARGb80xMRwiMXhzH8JhrF+4v9A0y74yE8ht3C/pp987xy21Ddzy1NvUvtOE9dftICC3PjI1ywiEtIRwXAxg5O+BGf/G7z+22BegzR3IcdjxrXnzOXrZx/Noy9tZ+mPn6GuoTUDBYuIBBQEw23RJ+EjP4Otq+G2M6B+c9pmn3zvHH50ybG8sm0PH/rhn3h9hya3F5HMUBBE4ehz4ZJ7gxC45TSoezVts9PmTuWXy46npb2LD//wv3n6L2Pj6lkRGV8UBFGZcyJc/ivobAvuQq5dkbZZdVU593/mPRxSms/Hb32W+1YNfhwjEZGDoSCI0rRq+NRvgstMf3YOvPZY2mZVEwu596r3cNzsifzPu9fwvcdeYyzdUiEiY5uCIGoTD4VP/RYmHQZ3XgRr707brKwgwU8vX8z5Cyv5t8de5e//Yy1tHRqTSESip8tHR0LxIXDZr+Cui+G+K2DHSzB1PuSXQUE55AeP3PwyvvuR+cyaVMh1v32VLbubufnSYykrGPrMaCIiB2Jj7RTEokWLfMWK9OfbR732Frj/b+ClPgdahdxiyC+n3gt5pT5GR24p1YfPpqh0UhgaZWFwlO3/Preoz7uaRSS7mdlKd1+Ubp2OCEZSIj+40axhG7Tshubd0FIfvG6p7/W+rHk3RxTuZNv2bexZ/wb5iRbi7QcYtTSW039QpL4vmgylM6BwkgJEJIspCEaaGZROCx4HUAbU7WjgI7c9x87GVr5/4TF84LDCA4ZIr/e739r3vquPoS/ieWFNM6B0evhIeV1UEdxBLSLjjk4NjQF1Da18+vYVrK3dzdfOOppPvnfO4HfiDu3NvYNibx3s2QJ7NofPW6AhfO5s6719LAeKp/YdFKXToWQqxMdAf0ZnO7Q2QFtj8NzaCO17gyOl0ukKPRmXdGpojKsoyeOuK5ZwzS+f51sPvcRbbzfxtbOPJh4bxOkcs2Dso9zC4MuuP+7BeEk9AZEUFHs2w/Z18Nqj0N6U+iFQPCVNWCSFRsm04BTZYLgHn9XauO/Lu60xeN/aAG0NSesaoXXPvtc9zw37vvQ7DzCkh8WDUCsJj9xKpqd/zi0a3M8hMkrpiGAM6exy/vnh9fzkqTc59agp3PDRGgpzM5Tl7sHRRU9AbEkTGlugtX7/bQsnJwXEtOCLt9cXfMP+X+I+wEtpc4uDR14J5CW97rUseV24PFEEze+ER0Rbg7mp92wJn7em/znyysJgmLYv5FIDo6gCYrpKWzKvvyMCBcEYdPvTG/jGgy8yd3oZt1y2iENKBvkX9khqbQi+SFOPKpJPQ3lXypdzui/svr7ES/e9ThRF96Xb2hh08vcERerz1mC9d/berueUWmpgpDynGalWZDhlJAjM7FbgbGCHu89Ls96A74/LbOUAAA1jSURBVAFnAk3AZe6+6kD7VRAEfrd+O3/7788zsSiX2y4/jndNKcl0SdLVua/fJfWIIjkwWvfsv21+WRAIJVODQMvJg5x8yMkNn/PSP8fz+l6Xk2ad+j6yVqaC4ESgEbi9jyA4E/g7giB4N/A9d3/3gfarINjnhdp6Pvmz52hp7+TmS47lhP8xOdMlyUC0NqYERdJz4/bgfpOOlqAvo6M1eN39fLBiOSlBcoCgSeRDTkEfz+EjUdDPc17QPq7uyEzL2KkhM5sNPNRHEPwIeMLd7wzfvwKc5O5b+9ungqC3zbubufy2P/NG3V7++cPH8JFFVZkuSaLiHlzN1RMMaYKi+3W6EOn13Me67v23t0BH875Q6n4MVSwnfZB0h0ZfoZNXEhwpFU/Zd8RUMEH3vQzBaL1qaAawKel9bbhsvyAws2XAMoCZM2eOSHFjxYzyAu656j185her+OI9a9n0dhOf/8C7MP1DGX/M9p3uyYSuriBg2pvDsGjuOzT2a5PuuXXfdi27999Pe3P6e1/iuUG/S8lUKJmSEhTd76dC4UQFxgBlMgjS/RdKe3ji7suB5RAcEURZ1FhUmp/gtsuP4x/vf4Ebfv86m95p5jvnH0Nejs4HyzCKxSBWEPwVP1LamqBxGzRsD06fNYbP3e93vgZv/jG4NyZVPDcMiKnpg6IkfBRMzPoruzIZBLVA8nmMSmBLhmoZ8xLxGP/3/PnMnFjIdx8NBqxbfukiygrHwA1eIn3JLQxG8J14aP/t2puDq7Z6gmLbvkfjNtj1Omx4KjjySBVL7AuM7kdyUJRMDS4Dzi8f/D0wY0Qmg+BB4G/N7C6CzuL6A/UPSP/MjL99/+FUTSzki/+xlg/d9Cd+etliZk7SpYkyziUKYOKc4NGf9uYwLFKCovv1rr/Axj8F95Skk1MQ9FH0PMrDx4Tej/yUZXklo/o0VWRBYGZ3AicBk82sFrgWSAC4+83AwwRXDL1OcPno5VHVkm3Oq5nB1NJ8lv18JR/64Z/4yScWsWDmhEyXJZJ5iQKYMDt49Ke9JSkwtgZ32je/Ez52h+N9vQNvv7FveX+d6RYfWGD0hEvS+hG44ko3lI1jf6lr5PLbnmP7nha+t7SG0+cdeKA7ERmi9uYgJHoC4519gZEcIqnv0921niyvNAyMclj4cVh8xZDKG61XDUnEDqso5v7PvIdP376Cq+5YxT+eeRSfeu8cXVEkEoVE2JE+gJGFe+nsCG4y7C8wugMloo56BcE4N6k4jzuvWMLnf7ma//2r9Wzc1cS15xxNTjy7r5IQGTXiOcGlroUTM1aCvg2yQH4izg8+tpC/OfFQfv7MRpb9fCV7WzsyXZaIjBIKgiwRixlfOfMo/tcH5/HEKzu48EdPs+nt1GGkRSQb6dRQlrl0ySwqywu4+t9X8b7/9zil+TnMmlTErEmFwWNi9+siDinJIzaYOQ9EZExSEGShk488hP/6u/fy+Ms72LBrLxt3NfHC5noeWbeNzq59V5Hl5cSYNamQmWE4zJ5UyMxJRcyaWMiMCQUk1M8gMi4oCLLUYRXFHFZR3GtZe2cXW3Y3s3FXExvDgNj4dvD6qdfraGnfNzlMPGbMKC/odSQxc1IhsycVMXNiIQW5Gt5CZKxQEEiPRDwWniYqAip6rXN3djS0smHnXja+3cRbu5rYsGsvb73dxIOrt7CnpXfn8yEleUEoTCpk1sRCZk0OjiRmTSqkvDB3BH8qETkQBYEMiJkxpTSfKaX5vPvQSfut393Utu8IIiks/vhqHTsaes8RXFaQCE85BcEwuTiPorwcinJzKMqLU5SXQ2FunOK8HApzcyjOyyE/EdP9DyIRURDIsCgvzKW8MJfqqvL91jW3dfLW2+ERxK4mNr4dnHZaW7t/v0RfYgZFuTkU5sXDwEgKi7wcivPiFIbLi3LjvZYVp7Qtyg3CRn0cIgEFgUSuIDfOEVNLOGLq/tNptnd20dDSwd7WDva2dbC3tZO9rR00db9OWha87mBvWydNrcHybXtaei9r60xTQXq58RhFPQESJy8nTiJu5ObESMRj5MbD5+73OUYidVm897LceIxETppl8VjPvnve5wTL8uLxnm1yYqYjHxlxCgLJqEQ8xsSiXCYWDU+/QVeX09ze2RMOe1s7wmDppDEMmMbWfaGRHDBtHV20dzptnV009rwPl4Wv2zq7aE9qF4XcMDQSOalhlBQwqYETT1qfs+99d2D1apMSdKmf1b2fnLiREzNiZuTEYsR7vbee9/GYEbfgWSE2NikIZFyJxSw4PZQX/f/a7k5HV0pIJIdGX0HSuW99W6eHwZKyrFe7lPfhNq3tXTS2dPS0b0vZT3u4vGMAp96GSzwMhpzucEgNi3gYKrGkUIknbdPziPWsj8foWRa34L9xd/DEkvbTve2+bWJhu97b9G4XLO/ZT087eu0znrJ96uf23rb3Z/SsS9kmZoya4FQQiAyRmfX89TyadXU57V1hMKSEVn+B1dUVBF1n0iN439VreUfSc1dKm66U9b331ZV2H+3tXXR2dfbaZ6f33rbL9z13b9vdrquLnvaj3X4BY5ATj/WEWU4sRixGT5uPLp7Jp993gEl6hkBBIDLOxWJGXixOXg6QoemOM8Hd6XJ6BUenO52d3YGxL2C6uqCjqyts13ubjuTg6Q4t3/c6XVB1prRJDq2uru7P6Aq3I1jXmfKZqft1Z3JxNP8BFQQiMi6ZGXEL/uqW/kV6TGtmp5vZK2b2upl9Oc36k8ys3sxWh4+vR1mPiIjsL8qpKuPAD4APEExU/5yZPejuL6U0fdLdz46qDhER6V+URwSLgdfd/Q13bwPuAs6L8PNERGQIogyCGcCmpPe14bJUx5vZGjN7xMzmptuRmS0zsxVmtqKuri6KWkVEslaUQZCuhyb1eq5VwCx3rwa+DzyQbkfuvtzdF7n7ooqKinRNRERkiKIMglqgKul9JbAluYG773H3xvD1w0DCzCZHWJOIiKSIMgieAw43szlmlgssBR5MbmBmUy28tc7MFof17IqwJhERSRHZVUPu3mFmfwv8BogDt7r7i2Z2Zbj+ZuAC4Coz6wCagaXuPvpvBxQRGUdsrH3vmlkdsHGIm08Gdg5jOWOdfh+96fexj34XvY2H38csd0/byTrmguBgmNkKd1+U6TpGC/0+etPvYx/9Lnob77+P0T1aloiIRE5BICKS5bItCJZnuoBRRr+P3vT72Ee/i97G9e8jq/oIRERkf9l2RCAiIikUBCIiWS5rguBAcyNkEzOrMrPHzWy9mb1oZp/LdE2ZZmZxM3vezB7KdC2ZZmblZnaPmb0c/j9yfKZryhQz+3z4b2Sdmd1pZvmZrikKWREESXMjnAEcDXzUzI7ObFUZ1QF8wd2PApYAV2f57wPgc8D6TBcxSnwP+LW7HwlUk6W/FzObAXwWWOTu8whGSFia2aqikRVBgOZG6MXdt7r7qvB1A8E/9HRDhGcFM6sEzgJ+kulaMs3MSoETgVsA3L3N3XdntqqMygEKzCwHKCRl4MzxIluCYKBzI2QdM5sNLACezWwlGXU98A9AV6YLGQUOBeqA28JTZT8xs6JMF5UJ7r4Z+C7wFrAVqHf3RzNbVTSyJQgGMjdC1jGzYuBe4Bp335PpejLBzM4Gdrj7ykzXMkrkAAuBm9x9AbAXyMo+NTObQHDmYA4wHSgys0syW1U0siUIDjg3QrYxswRBCNzh7vdlup4MOgE418w2EJwyfL+Z/SKzJWVULVDr7t1HiPcQBEM2OhV4093r3L0duA94T4ZrikS2BMEB50bIJuEcELcA6939ukzXk0nu/hV3r3T32QT/X/ze3cflX30D4e7bgE1mdkS46BTgpQyWlElvAUvMrDD8N3MK47TjPLL5CEaTvuZGyHBZmXQCcCnwgpmtDpd9NZwlTuTvgDvCP5reAC7PcD0Z4e7Pmtk9BFPqdgDPM06HmtAQEyIiWS5bTg2JiEgfFAQiIllOQSAikuUUBCIiWU5BICKS5RQEIiPIzE7SCKcy2igIRESynIJAJA0zu8TM/mxmq83sR+F8BY1m9q9mtsrMfmdmFWHbGjN7xszWmtn94Rg1mNn/MLPHzGxNuM1h4e6Lk8b7vyO8a1UkYxQEIinM7CjgIuAEd68BOoGLgSJglbsvBP4AXBtucjvwJXefD7yQtPwO4AfuXk0wRs3WcPkC4BqCuTEOJbjTWyRjsmKICZFBOgU4Fngu/GO9ANhBMEz1L8M2vwDuM7MyoNzd/xAu/xnwH2ZWAsxw9/sB3L0FINzfn929Nny/GpgNPBX9jyWSnoJAZH8G/Mzdv9JrodnXUtr1Nz5Lf6d7WpNed6J/h5JhOjUksr/fAReY2SEAZjbRzGYR/Hu5IGzzMeApd68H3jGz94XLLwX+EM7vUGtmHwz3kWdmhSP6U4gMkP4SEUnh7i+Z2T8Bj5pZDGgHriaYpGWuma0E6gn6EQA+AdwcftEnj9Z5KfAjM/tWuI+PjOCPITJgGn1UZIDMrNHdizNdh8hw06khEZEspyMCEZEspyMCEZEspyAQEclyCgIRkSynIBARyXIKAhGRLPf/Aalxq3mzSR8wAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# chart accuracy and loss for train and validation sets\n",
    "\n",
    "print(modelfit.history.keys())\n",
    "#  acc\n",
    "#plt.plot(modelfit.history['accuracy'])\n",
    "plt.plot(modelfit.history['val_accuracy'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "# plt.legend(['train', 'validation'], loc='upper left')\n",
    "#plt.legend(['validation'], loc='upper left')\n",
    "plt.show()\n",
    "# Loss\n",
    "plt.plot(modelfit.history['loss'])\n",
    "plt.plot(modelfit.history['val_loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'validation'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Confusion matrix <a name='confusionmatrix' />\n",
    "<a href=#linkanchor>Back to link list</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEWCAYAAACOv5f1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxU1d3H8c8vARSVfQ0BxQU3XGhtlce6VhTcCj6tilZBq0WRUrdqAX1csLS4Vm1VihsoKuJWcUGkWLVWLKK1LlArAkogsgkIaIEkv+ePe4I3YTKZgYRJcr9vXveVmXPvPefc7XfPnHtmMHdHRESSJS/XFRARkW1PwV9EJIEU/EVEEkjBX0QkgRT8RUQSSMFfRCSBchr8zaypmT1nZqvN7ImtyOenZvZyTdYtF8xsipkN3MJ1f2Nmy83si5quV5KY2XVmNiHDZV81s/Nru041zcwWmFmv8HqEmd23hfl8ZGZH1Wjlvs37AjO7vYbzzLi+8X1UGyodg1+a2ejaKqsqGQV/MzvTzGaZ2VozKw5B6rAaKP8nQAegjbufuqWZuPsj7n5cDdSnAjM7yszczJ6ulH5gSH81w3wyCijufry7j9+CenYBLgf2dfeO2a6fZVnjzOw3tZj/OWb2Rm3lLxW5+2/dvdobWKrj7u7d3f3Vmq6TmTUBrgZursl8a6q+IS4U1UCVyo0FzjKz9jWYZ7WqDf5mdhlwO/BbokC9M3A30LcGyt8F+I+7l9RAXrVlGXCombWJpQ0E/lNTBVhkaz6F7QKscPelW1B2o60ot9bzq0lmlp/rOtS0ury/t0Jf4N/uvijXFdkW3P2/wBRgwLYuuMoJaAGsBU5Ns8x2RDeHxWG6HdguzDsKKCJqlS4FioFzw7zrgQ3AxlDGecB1wIRY3l0BBxqF9+cA84A1wHzgp7H0N2LrHQq8DawOfw+NzXsVuAH4e8jnZaBtFdtWXv8xwJCQlh/SrgFejS17B7AQ+Ap4Bzg8pPeptJ3/itVjVKjHN8AeIe38MP8e4MlY/jcC0wGrVMdeYf2ykP+4kP4j4CNgVch3n9g6C4BfA+8D68v3b2y+Ab8Px2x1WG4/YFDYjg2hrOeqyg/oCbwZyv8XcFSl8+r+cD4sAn4T9us+wH+B0pD/qrB8U+BW4LNQnzeApmHeE8AXIf11oHusnHFhP74IrAN6pTjGuwKvhXNhGvBHKp6D6bYjfrx2B14BVgDLgUeAlmHeFcBTlcr9A3B7FefdAmA4MBtYCTwIbF/pnPx12O6HiRpxw4BPQ/mTgNax/M4O+24FcFXIv1eYd12l7T0str0Lia6tdMe9PJ8tigNVbP8DwNWx9+OBy8PrQqKYcFF4vwfwJeG6AE4C3gv1fxM4oNJ+La9v05DvSmAOcCVQVGnZXxGd06uBx4HtgR2peL2tBTptzTEI838K/DVdPK7pqbrg3wcooVJwqLTMSOAtoD3QLuzwG2IHvSQs0xg4AfgaaFXFiVf5fddwoBuFnf4VsFeYV0C40IkFf6B1OKBnh/XOCO/bxC7YT4E9wwnwKjC6muB/KPCPkHYCMBU4n4rB/yygTSjzcqILc/tU2xWrx+dA97BOYyoGkx2IPl2cAxxOFFA6p6tn7P2eRMHu2JDvlcBcoEnsxH4P6EIIopXy6010A2tJdCPYByiIBdTfpAhWm/IjukBXhH2VF+qxAmgXlv8z8KdwTNsDM4ELKh/LWP53hX1TSHSTOJRvA8vPgGZ8G3zei603jujC/UGox/YptnUGcFtY/wiim8CEWKBJtx3x47VHmL8d0XXwOiG4E52r6/j2ZtCIKAgeVMXxXAB8GPZna6IGwm8qXVM3hrKaApcQXYOdQ9qfgMfC8vsSBagjwrzbwvqbBX+iT/VriK6ZxkTnc49qjnt5PlscB1Js/9vEGpzhGJffcM4kun4fj817Nrz+btivhxCdJwNDHbdLUd/RRDf9VmG/vc/mwX8mUWBvTXSDuDDV9RbStvgYxOr+ZW0G+832c9qZ0d3oi2qW+RQ4oVLgWBDbSd8Qu3mEg9MzVVBM8b4rFYP/KuDHVApYVAz+ZwMzU1zg58Qu2Hir4iLgpSq2bdNBBj4B9gImhv1SIfinWHclcGCq7YrVY2SKtPNj7w8matV8BpyRpqwKJyPwf8Ck2Ps8ohb2UbET+2dp8vsh0Y2nJ5BXad44UgeBn8Xe/xp4uNIyU4kuxg5Enw6axuadQWj1sPmnuLxwDh1Y7ckc3awcaBGr60Nplt+Z6CLcMZb2KN8Gwyq3I9XxqrRcP+CfsfdTgJ+H1ycBs9PUawEh0IT3JwCfxo71BmI3MqLAdEzsfQFRS70R0SfUibF5O4b1UwX/4cAzVdSpquNens8Wx4EUZX0C9Im9353o2s8j+hR+Ad9el+OBy8Lrewg3nNi6HwNHpqjvPKB3bLnz2Tz4nxV7fxMwJtX1trXHIKR1A0qrO8drcqqun3kF0LaafsVORMGp3GchbVMeXrFP/2tgp2rK3Yy7rwNOBy4Eis3sBTPbO4P6lNepMPY+PiIm0/o8DPwCOBp4pvJMM7vczOaEkUuriLo22laT58J0M919JtFJakQfIzNVYR+4e1koK74Pqizb3V8h6v64C1hiZmPNrHk1Zcbz2wU41cxWlU9E3QkFYV5jomNYPu9PRC3GVNoSfdz+tPIMM8s3s9Fm9qmZfUV0wZavU+12Eu2nleHcKhc/d9JtR+W6tDeziWa2KNRlQqV6jCf6dEj4+3CaelWud+VraplH/cTxej4Tq+Mcoq6zDmG9TXmFbV1RRZldSLGfM1STcWAl0ac5ANz9U6KWcw+iT8HPA4vNbC/gSKIWPET74fJKx6tLpXrE6xvfx6nOk2zixNYeg2ZEn1K3meqC/wyiPth+aZZZTLTh5XYOaVtiHVF3R7kKI1fcfaq7H0t08f0buDeD+pTXaWsfHj1M9CnhRXf/Oj7DzA4naiWeRvRRtiXRgbTyqleRZ1Xp5fkOIfqYuJio6yZTFfaBmRnRRRDfB2nLdvc73f0gom6pPYn6rdOtF09fSNRibhmbdnT30WHeeqLnLOXzmrt79yryX050Du6eoswziR4O9iK62XYt3+QMt7MYaGVmO8bSds5wOyr7XSjrAHdvThTg4/X4M3CAme1H1PJ/JE29IDpe8TrFr6nK27QQOL5SPbf36IFpcTwvM9uBqDsnlYWk3s+pyqysJuPA+0TnXNxrRKMDm4Tteo3oAWkroi5HiOo/qtJ+2MHdH0tRRjFRF025LimWqUqqfbG1x2AfomdK20za4O/uq4k+stxlZv3MbAcza2xmx5vZTWGxx4CrzaydmbUNy2c0TjqF94AjzGxnM2tB9DEUADPrYGY/ChfqeqKWQGmKPF4E9gzDUxuZ2elEfW7Pb2GdAHD3+UStjKtSzG5G1H2wDGhkZtcA8ZbyEqBrNiN6zGxPogehZxF1ZV1pZj0yXH0ScKKZHWNmjYmeQawn6ofNpOzvm9khYd11fPsQtnxbdqsmiwnAyWbWO7TOtw/D4zq7ezHRQ/Zbzay5meWZ2e5mdmQs/85huF/5p5YHgNvMrFPI73/MbDui/b6eqBW1A9GItIy5+2fALOB6M2sShi+fnMl2pMiuGeEhtZkV8u3Nsrys/wJPEnUrzXT3z6up3hAz62xmrYERRA8cqzIGGGVmuwCEa7F8NN6TwElmdljYpyOp+rp/BOhlZqeFa6dN7Jyr7rjXZBx4kehai3uN6JP36+H9q8BQoi7C8nPzXuDCcO6ame1oZieaWTM2NwkYbmatwvH6RRb1WwK0CTGq3NYegyOJuga3mWqDkbvfBlxGNO52GdEd7hdELRmIAtQsorv1B8C7IS1r7j6N6CR/n+iBYzxg5xEFscVE/eBHErXEK+exgqhldTlRULgSOMndl29JnSrl/Ya7p2rNTCU6cP8h+rj7Xyp+jCz/AtsKM3u3unJCN9sE4EZ3/5e7f0IUAB4OQa+6en5MdNP4A1HL+WTgZHffUN26QXOiC2kl345QuCXMux/YN3y8/XOqld19IVGLfATfnjNX8O35NgBowrejWZ7k266UV4hGKX1hZuXH7FdE59bbRMf+xpDXQ6F+i0Jeb2W4fXFnEj0g/BK4NuSZ6XbEXU/00G418ALwdIplxgP7U32XD0Q3iZeJuv3mkf6augOYDLxsZmuI9sMhYRs+AoaE/IqJ9nfKMerhhnQC0bXzJVFj7MAwu7rjXmNxAHgO2NvM4t01rxHdYMuD/xtEN/zy97j7LODnRF2WK4kGOZxTRRkjifbDfOAvROfg+kwq5+7/JrrZzQv7oxNbcQzMbHui/Z71d3y2RvnwKBGpZWa2M1F3ZUd3/yrNcguIHiT/ZVvVra4xs0FEX1q8ZBuVNxjo7+6VP3Fsi7KHAl3cPZuu3a3WEL8gIlLnhC6/y4hGfVQZ+CXi7mNrM38zKyDqxppBNNLmcqJPDNucu/8hF+Uq+IvUsvCcaglRF1WfHFdHIk2IRpntSjSMdCLRLxckhrp9REQSSD/pLCKSQPW+22fj8nn66CKbadrp8FxXQeqgkg2LrPql0ssm5jRuu9tWl1db1PIXEUmget/yFxHZpspSfbe0/lHwFxHJRmld/u9HMqfgLyKShegXR+o/BX8RkWyUKfiLiCSPWv4iIgmkB74iIgmklr+ISPK4RvuIiCSQHviKiCSQun1ERBJID3xFRBJILX8RkQTSA18RkQTSA18RkeRxV5+/iEjyqM9fRCSB1O0jIpJAavmLiCRQ6cZc16BGKPiLiGRD3T4iIgmkbh8RkQRqIC3/vFxXQESkXikry3yqhpm1NLMnzezfZjbHzP7HzFqb2TQz+yT8bRVbfriZzTWzj82sdyz9IDP7IMy708ysurIV/EVEsuClGzOeMnAH8JK77w0cCMwBhgHT3b0bMD28x8z2BfoD3YE+wN1mlh/yuQcYBHQLU5/qClbwFxHJhpdlPqVhZs2BI4D7Adx9g7uvAvoC48Ni44F+4XVfYKK7r3f3+cBc4GAzKwCau/sMd3fgodg6VVLwFxHJRhbdPmY2yMxmxaZBsZx2A5YBD5rZP83sPjPbEejg7sUA4W/7sHwhsDC2flFIKwyvK6enpQe+IiLZyGK0j7uPBcZWMbsR8F1gqLv/w8zuIHTxVCFVP76nSU9LLX8RkWzU3APfIqDI3f8R3j9JdDNYErpyCH+XxpbvElu/M7A4pHdOkZ6Wgr+ISDZqqM/f3b8AFprZXiHpGGA2MBkYGNIGAs+G15OB/ma2nZntSvRgd2boGlpjZj3DKJ8BsXWqpG4fEZFslNTof+YyFHjEzJoA84BziRrlk8zsPOBz4FQAd//IzCYR3SBKgCH+7e9LDwbGAU2BKWFKS8FfRCQbNfgNX3d/D/heilnHVLH8KGBUivRZwH7ZlK3gLyKSjQbyDV8FfxGRbOi3fUREEkgtfxGRBFLLX0QkgWp2tE/OKPiLiGTDq/3ybL2g4C8ikg31+YuIJJCCv4hIAumBr4hIApWWVr9MPaDgLyKSDXX7iIgkkIK/iEgCqc9fRCR5vEzj/EVEkkfdPiIiCaTRPiIiCaSWv4hIAin4y9b6as1arh19O3PnfQZm3DDiUnrst8+m+WvWrmPYyJsoXrKM0pJSzjnzx5xy4nFbVeaGDRsYfsOtzP74E1q2aM4tI4dTWNCBxV8s4ZIRv6G0tIySkhLO/MmPOP2UE7d2EyXHeh93FLfdNpL8vDweePAxbrr5rlxXqf5rID/slpfrCiTZ6NvH8INDvsdzj93L0+PvYrddulSY/9hTz7F71515evzdPPjHG7n5D/eycePGjPJeVLyEc35x5WbpTz//Ms2b7cSUSQ9w9un9uO3uBwBo16Y1E8bcylPj7+Kxe2/n/gmTWLpsxdZvpORMXl4ed94xipNOPov9Dzya00/vxz77dMt1teq/srLMpzos5y1/M9sb6AsUAg4sBia7+5ycVqyWrV23jnf+9SGjrr4cgMaNG9O4ceMKy5gZ677+Bnfn62/+S4vmzcjPzwfguamv8MgTz7JxYwkHdN+Lqy8fsmleOq/8bQYXnXcWAMcddTi/ve0e3L1C2Rs2bqSsgbRukuzg73+HTz9dwPz5nwMwadKz/Ojk3syZ80mOa1bPNZChnjlt+ZvZr4GJgAEzgbfD68fMbFgu61bbihZ9QauWLbh61G385JwhXPO72/n6m/9WWObMH5/MvAULObrvTzllwGCGXXIheXl5fLrgc16a/hoPh5Z6Xl4ez7/814zKXbpsBR3btwWgUaN8dtpxB1at/gqA4iXLOGXAYHqdMoDzfnoq7du1qdmNlm2qU2FHFhYt3vS+aFExnTp1zGGNGojS0synOizXLf/zgO7uXqEvw8xuAz4CRqdaycwGAYMA7r71N5w/4IzarmeNKyktZc5/5jLi0sEc0H1vfnf7GO5/eBJDBw3YtMzfZ77D3t1244E/jGbhomJ+fskIDjqwO/+Y9R6z/z2X/uddDMD69etp3aolAL8cPpJFi5ewsWQjxUuW8eOBQwA467S+nHLicXiKFr2ZAVDQoR3PPHQPS5et4JfDR3Ls0YfRtnWr2t4VUkvKj2tcquMv2fEa7s4xswXAGqAUKHH375lZa+BxoCuwADjN3VeG5YcTxc5S4JfuPjWkHwSMA5oCLwIXe5oDnuvgXwZ0Aj6rlF4Q5qXk7mOBsQAbl8+rl2dzx/Zt6dCuLQd03xuA4446jPsmTKqwzDMvTOP8s07DzNi5cycKCzoy/7Mi3J0fHd+LSwefu1m+d/7uGiDq879q1K2M++NNFeZ3aN+WL5Yup2P7dpSUlLJ23de0aN6swjLt27Vhj1134d1/fchxRx9ek5st29CiomK6dO606X3nwgKKi5fksEYNRO10+xzt7stj74cB0919dOgFGQb82sz2BfoD3Yli51/MbE93LwXuIWoUv0UU/PsAU6oqMNcPfC8BppvZFDMbG6aXgOnAxTmuW61q26Y1Hdu3Y/5nRQC89c577N515wrLFHRox1vvvAfA8i9XsuDzIjp36kjP7/Vg2qtvsGLlKgBWf7WGxV9kdlEffVhPnn3xLwC8/OrfOOSgAzEzvli6jP+uX78pv39+MJuuO3eukW2V3Hh71nvssceudO3ahcaNG3PaaX157vmXc12t+s/LMp+2XF9gfHg9HugXS5/o7uvdfT4wFzjYzAqA5u4+I7T2H4qtk1JOW/7u/pKZ7QkcTPTA14Ai4O1wJ2vQRlw6mF9ffxMbSzbSpVMBN4y4lMefeQGA0085kQvPOZOrRt3KKWcPxt259KKf0aplC1q1bMHQnw9g0CVXUeZlNG7UiKsuu4hOHTtUW+b/ntSb4TfczPGn/YwWzZtx8/XRo5V5CxZy8x/vxcxwd84543/Zc/dda3X7pXaVlpZy8SVX8+ILj5Kfl8e48Y8ze/Z/cl2t+q/mW/4OvGxmDvwp9Gx0cPdiAHcvNrP2YdlCopZ9uaKQtjG8rpxeJavvfYD1tdtHalfTTuquks2VbFi0+YOQLK27pn/GMWenGx6/gPB8MhgbgvsmZtbJ3ReHAD8NGEo04rFlbJmV7t7KzO4CZrj7hJB+P1EXz+fA79y9V0g/HLjS3U+uqm657vMXEalfsujOiT+fTLPM4vB3qZk9Q9QTssTMCkKrvwBYGhYvAuJfCOpMNDy+KLyunF6lXPf5i4jUL2We+VQNM9vRzJqVvwaOAz4EJgMDw2IDgWfD68lAfzPbzsx2BboBM0MX0Roz62nRMK8BsXVSUstfRCQLNTzUswPwTBiW2wh4NDwLfRuYZGbnEXXpnArg7h+Z2SRgNlACDIk9Hx3Mt0M9p5BmpE95YSIikqkafODr7vOAA1OkrwCOqWKdUcCoFOmzgP0yLVvBX0QkGw3k5x0U/EVEslHHf7YhUwr+IiJZ0P/hKyKSRAr+IiIJVMd/pz9TCv4iItlQy19EJIEU/EVEksdL1e0jIpI8avmLiCSPhnqKiCSRgr+ISAI1jC5/BX8RkWx4ScOI/gr+IiLZaBixX8FfRCQbeuArIpJEavmLiCSPWv4iIkmklr+ISPJ4Sa5rUDMU/EVEsuBq+YuIJJCCv4hI8qjlLyKSQA0l+OflugIiIvWJl1rGUybMLN/M/mlmz4f3rc1smpl9Ev62ii073MzmmtnHZtY7ln6QmX0Q5t1pZtUWruAvIpIFL8t8ytDFwJzY+2HAdHfvBkwP7zGzfYH+QHegD3C3meWHde4BBgHdwtSnukIV/EVEsuBllvFUHTPrDJwI3BdL7guMD6/HA/1i6RPdfb27zwfmAgebWQHQ3N1nuLsDD8XWqZKCv4hIFrJp+ZvZIDObFZsGVcruduBKKo4h6uDuxQDhb/uQXggsjC1XFNIKw+vK6Wnpga+ISBbcM+vLj5b1scDYVPPM7CRgqbu/Y2ZHZZBdqoI9TXpaCv4iIlmowdE+PwB+ZGYnANsDzc1sArDEzArcvTh06SwNyxcBXWLrdwYWh/TOKdLTUrePiEgWykot4ykddx/u7p3dvSvRg9xX3P0sYDIwMCw2EHg2vJ4M9Dez7cxsV6IHuzND19AaM+sZRvkMiK1TJbX8RUSykMmD3K00GphkZucBnwOnArj7R2Y2CZgNlABD3L00rDMYGAc0BaaEKS2LHg7XXxuXz6vfGyC1ommnw3NdBamDSjYs2urIvaDHsRnHnK7vTav1O8WWUstfRCQL9by9vImCv4hIFrZBt882oeAvIpKFbIZ61mUK/iIiWSjN8Dd76joFfxGRLKjlLyKSQOrzFxFJII32ERFJILX8RUQSqLSsYfwqjoK/iEgW1O0jIpJAZRrtIyKSPBrqKSKSQOr2qSOO63FBrqsgddC5nQ7NdRWkgVK3j4hIAmm0j4hIAjWQXh8FfxGRbKjbR0QkgTTaR0QkgcpyXYEaouAvIpIFRy1/EZHEKVG3j4hI8qjlLyKSQA2lz79hfFtBRGQbcSzjKR0z297MZprZv8zsIzO7PqS3NrNpZvZJ+Nsqts5wM5trZh+bWe9Y+kFm9kGYd6eZVfvxRMFfRCQLZVlM1VgP/NDdDwR6AH3MrCcwDJju7t2A6eE9ZrYv0B/oDvQB7jaz/JDXPcAgoFuY+lRXuIK/iEgWSrGMp3Q8sja8bRwmB/oC40P6eKBfeN0XmOju6919PjAXONjMCoDm7j7D3R14KLZOlRT8RUSyUGaZT2Y2yMxmxaZB8bzMLN/M3gOWAtPc/R9AB3cvBgh/24fFC4GFsdWLQlpheF05PS098BURyUJZFqN93H0sMDbN/FKgh5m1BJ4xs/3SZJeqYE+TnpZa/iIiWfAspozzdF8FvErUV78kdOUQ/i4NixUBXWKrdQYWh/TOKdLTUvAXEclCTT3wNbN2ocWPmTUFegH/BiYDA8NiA4Fnw+vJQH8z287MdiV6sDszdA2tMbOeYZTPgNg6VVK3j4hIFsqqH0WZqQJgfBixkwdMcvfnzWwGMMnMzgM+B04FcPePzGwSMBsoAYaEbiOAwcA4oCkwJUxpKfiLiGShtPpFMuLu7wPfSZG+AjiminVGAaNSpM8C0j0v2IyCv4hIFsoaxq87KPiLiGQjm9E+dZmCv4hIFvTfOIqIJJC6fUREEqih/Kqngr+ISBZK1fIXEUketfxFRBJIwV9EJIEayH/hq+AvIpINtfxFRBKopn7eIdcU/EVEsqBx/iIiCaRuHxGRBFLwFxFJIP22j4hIAqnPX0QkgTTaR0QkgcoaSMePgr+ISBb0wFdEJIEaRrtfwV9EJCtq+YuIJFCJNYy2v4K/iEgWGkboh7xcV0BEpD4py2JKx8y6mNlfzWyOmX1kZheH9NZmNs3MPgl/W8XWGW5mc83sYzPrHUs/yMw+CPPuNLNqv42g4C8ikoUyPOOpGiXA5e6+D9ATGGJm+wLDgOnu3g2YHt4T5vUHugN9gLvNLD/kdQ8wCOgWpj7VFa7gLyKSBc9iSpuPe7G7vxterwHmAIVAX2B8WGw80C+87gtMdPf17j4fmAscbGYFQHN3n+HuDjwUW6dKCv4iIlnIptvHzAaZ2azYNChVnmbWFfgO8A+gg7sXQ3SDANqHxQqBhbHVikJaYXhdOT0tPfAVEclCaRaPfN19LDA23TJmthPwFHCJu3+Vprs+1QxPk56WWv4iIlmoqQe+AGbWmCjwP+LuT4fkJaErh/B3aUgvArrEVu8MLA7pnVOkp6XgLyKSBc/iXzphRM79wBx3vy02azIwMLweCDwbS+9vZtuZ2a5ED3Znhq6hNWbWM+Q5ILZOldTtIyKShRr8hu8PgLOBD8zsvZA2AhgNTDKz84DPgVMB3P0jM5sEzCYaKTTE3ct/ZHQwMA5oCkwJU1oK/jl05S2X07PXIaxavoqf9dr8OVCvU35I/4tOB+Cbdd9w+/A7+XTOvK0qs3GTxgy//Ur2PKAbX638iusHj2JJ0RI6FLZn5L3XkpefT6NG+Tz94LM8N+H5rSpLtszAmwZzwA8PYs2K1VzX+/LN5jdttgPn/X4orQvbkp+fz9R7J/PmE69uVZmNmjTiZ7cNZZf9dmPtqjWM/cXvWVG0jNaFbblozBXk5eeR3yifV8ZP4bVHpm1VWfVdTf2qp7u/Qer+eoBjqlhnFDAqRfosYL9syle3Tw699MTL/PqsEVXOL/78Cy75yeWcf+wFPHzHI1x+0yUZ592hcwd+/8Qtm6Wf0L8Pa1av5azDzuGJe5/mghHnA7Bi6Zf8ot8l/Lz3hQw+eShnDjmdNh3aZL9RstXefPJV7hi42fW9ydFn96Z4bhEjj7+Cm/tfx2lXDSS/cWbtuDad2/Griddtln7YaT/k69Vrueqoofzl/uf58bCzAFi9dBWjf3wVI0+4gt/2G0Gfwf1o0b7VZusnSU0N9cw1Bf8cev8fH/DVqjVVzv/ondmsXb0WgNnvzqFtQbtN83r97zHc/fwfuHfqGC4bfTF5eZkdyh8cdyhTn3gZgNdeeJ3vHvYdAEo2lrBxw0YAmjRpjGWYn9S8T2bOYV047qk4znY7NgVg+x22Z92qtZSVRJ/+D+l3OCP+/DuuefFmzvrtoIyPY4/jvs+bT70GwDsvvsXeh0aNyNKNJZRsKAGiTwdmOi9K8Iynug+mQckAAAsCSURBVExHsp44oX8fZv71bQB23mNnjj75SIaGlnpZaRm9TvlhRvm07diGpcXLACgrLWPtV+to3qo5AO0K2nHftD/x+NuPMvHux1mxZEXtbIxslVfGv0TBHoXcPHMs1069lYnXP4i703H3Qr5/0qHc+JOrGXnCFZSVltGz32EZ5dmyQ2tWLl4OROfFN2u+ZqdWzQBoVdCGa6fcwo0zxvDSmD+zeunKWtu2+qCmHvjmWp3t8zezc939wSrmDSL6KjN7ttybTjt2TrVYg9Hj0AM5of/x/PKUqNvnu4d9hz3335MxL9wFQJPtm7ByxSoARt53LQVdCmjUuBEdCttz79QxADx1/zO8NGkqKccQe3SSLitexvnHXkCbDm244f7reO2F11m5fNU22ELJRvcjerBw9gJuPeN62u3Skcsm/B/XHz+HfX6wP7vsvxtXTR4NQOPtmrBmxWoALvrTFbTt0p78xo1o3akt17x4MwB/efAF3nzi1ZTnhYfzYmXxCq4//le0aN+KIWOv5J0pb7Fm+epttLV1j37SufZdD6QM/vEvThzd+di6fXvdSrvtsyu/uukyhp09YlMXkRlMffJl7hv9wGbLX3P+9UDU5z/s91dw6am/qjB/WfFy2he0Y3nxcvLy89ip+Y6bdT2tWLKCBR9/xv6H7M/rL/ytlrZMttQPTj2al+55BoBln33B8oVL6bh7IWbw5lOv8cxNj262zt0XRMG+Ted2nHvLEG7pf12F+Su/WEGrTm1Z+cWX5OXn0bTZDqxbVbHrafXSlSz+ZCHdvr8P7055q3Y2rh6o6y36TOW028fM3q9i+gDokMu61QXtO7Vj5L3X8ruLb6Ro/qJN6e++8U+OPPEIWrZpCUCzls3oUNi+qmwqeHPaDHqfehwAR554BP/8ezTCrG1BW5ps3wSAnVrsxH7f787CTxdWmY/kzpeLl7P3D/YHoFnbFnTYrRPLP1/CnL9/yEHH96RZm6gbb4cWO9G6sG1Geb43bRaH/vhIAA46oScfv/khAK06tqbxdtF5sUPzHdnjoL1YMq/a7w81aDX5Ja9cynXLvwPQG6jciWjAm9u+OtvW1X8cQY//OYAWrVsw6e1HGXfrQ+Q3ig7JcxOeZ8ClZ9O8ZXMu+e0vASgtKeXCE4fw2Sef88BND3Lzo6OxPKN0Ywm3X/1Hlixamq44AF6YOIURdwxjwhvj+GrVGm64KBpVssseOzP4mguiLiAzJv3pCeb/e0GtbbtU7ed3XsyePbuzU6tm3DRjDJN/P4n8xtGPN772yDSev/NJzr1lCNe+dCtm8NToCaxduYa1K9fw51sncunD/4eZUVpSyqPX3MeXi5ZXW+Ybk17hvNuGMurVP7Bu1VrGDv09AB336MxpVw3AcQxj6r3Psejjz2t1++u6Um8YLX/zHG6Imd0PPBjGu1ae96i7n1ldHg2920e2zB6NWua6ClIH3bvgiWp/5746Z+5ySsYx59HPntnq8mpLTlv+7n5emnnVBn4RkW2tofT557rbR0SkXqnrffmZUvAXEclCTf28Q64p+IuIZEHdPiIiCdRQRvso+IuIZEHdPiIiCaQHviIiCaQ+fxGRBFK3j4hIAuXyVxFqkoK/iEgWStXyFxFJHnX7iIgkkLp9REQSSC1/EZEEaihDPfUfuIuIZKHUPeOpOmb2gJktNbMPY2mtzWyamX0S/raKzRtuZnPN7GMz6x1LP8jMPgjz7rSU/1l3RQr+IiJZKMMznjIwDuhTKW0YMN3duwHTw3vMbF+gP9A9rHO3meWHde4BBgHdwlQ5z80o+IuIZKEmg7+7vw58WSm5LzA+vB4P9IulT3T39e4+H5gLHGxmBUBzd5/h0dPoh2LrVEnBX0QkC+6e8WRmg8xsVmwalEERHdy9OJRVDLQP6YXAwthyRSGtMLyunJ6WHviKiGQhm9E+7j4WGFtDRafqx/c06Wmp5S8ikgXP4t8WWhK6cgh/l4b0IqBLbLnOwOKQ3jlFeloK/iIiWSj1soynLTQZGBheDwSejaX3N7PtzGxXoge7M0PX0Boz6xlG+QyIrVMldfuIiGShJr/ha2aPAUcBbc2sCLgWGA1MMrPzgM+BU0O5H5nZJGA2UAIMcffSkNVgopFDTYEpYUpLwV9EJAs1+Q1fdz+jilnHVLH8KGBUivRZwH7ZlK3gLyKShYbyDV8FfxGRLJTph91ERJJHLX8RkQTailE8dYqCv4hIFtTtIyKSQOr2ERFJILX8RUQSSC1/EZEEKt30pdr6TcFfRCQL+g/cRUQSSP+Bu4hIAqnlLyKSQBrtIyKSQBrtIyKSQPp5BxGRBFKfv4hIAqnPX0QkgdTyFxFJII3zFxFJILX8RUQSSKN9REQSSA98RUQSqKF0++TlugIiIvWJZ/GvOmbWx8w+NrO5ZjZsG1R/E7X8RUSyUFMtfzPLB+4CjgWKgLfNbLK7z66RAqqh4C8ikoUa7PM/GJjr7vMAzGwi0BdQ8M/EX4umWa7rUFeY2SB3H5vrekjdovOiZpVsWJRxzDGzQcCgWNLY2LEoBBbG5hUBh2x9DTOjPv+GZVD1i0gC6bzIEXcf6+7fi03xm3Cqm8g2e5qs4C8ikhtFQJfY+87A4m1VuIK/iEhuvA10M7NdzawJ0B+YvK0Kr/d9/lKB+nUlFZ0XdZC7l5jZL4CpQD7wgLt/tK3Kt4byhQUREcmcun1ERBJIwV9EJIEU/BuIXH5NXOomM3vAzJaa2Ye5rovUPQr+DUDsa+LHA/sCZ5jZvrmtldQB44A+ua6E1E0K/g3Dpq+Ju/sGoPxr4pJg7v468GWu6yF1k4J/w5Dqa+KFOaqLiNQDCv4NQ06/Ji4i9Y+Cf8OQ06+Ji0j9o+DfMOT0a+IiUv8o+DcA7l4ClH9NfA4waVt+TVzqJjN7DJgB7GVmRWZ2Xq7rJHWHft5BRCSB1PIXEUkgBX8RkQRS8BcRSSAFfxGRBFLwFxFJIAV/yQkzKzWz98zsQzN7wsx22Iq8xpnZT8Lr+9L9qJ2ZHWVmh8beX2hmA7a0bJH6SsFfcuUbd+/h7vsBG4AL4zPDL5Vmzd3Pd/fZaRY5CtgU/N19jLs/tCVlidRnCv5SF/wN2CO0yv9qZo8CH5hZvpndbGZvm9n7ZnYBgEX+aGazzewFoH15Rmb2qpl9L7zuY2bvmtm/zGy6mXUluslcGj51HG5m15nZr8LyPczsrVDWM2bWKpbnjWY208z+Y2aHb9O9I1IL9B+4S06ZWSOi/4fgpZB0MLCfu883s0HAanf/vpltB/zdzF4GvgPsBewPdABmAw9UyrcdcC9wRMirtbt/aWZjgLXufktY7pjYag8BQ939NTMbCVwLXBLmNXL3g83shJDeq6b3hci2pOAvudLUzN4Lr/8G3E/UHTPT3eeH9OOAA8r784EWQDfgCOAxdy8FFpvZKyny7wm8Xp6Xu6f9XXszawG0dPfXQtJ44InYIk+Hv+8AXTPbRJG6S8FfcuUbd+8RTzAzgHXxJKKW+NRKy51A9T9ZbRksk4314W8pum6kAVCfv9RlU4HBZtYYwMz2NLMdgdeB/uGZQAFwdIp1ZwBHmtmuYd3WIX0N0Kzywu6+GlgZ688/G3it8nIiDYVaMFKX3UfUxfKuRR8LlgH9gGeAHwIfAP8hRZB292XhmcHTZpYHLAWOBZ4DnjSzvsDQSqsNBMaEYafzgHNrY6NE6gL9qqeISAKp20dEJIEU/EVEEkjBX0QkgRT8RUQSSMFfRCSBFPxFRBJIwV9EJIH+H38uYIDuNQKOAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "\n",
    "cfmap=metrics.confusion_matrix(y_true=test['target'],  # True labels\n",
    "                         y_pred=test[\"predround\"])\n",
    "\n",
    "label = [\"0\", \"1\"]\n",
    "sns.heatmap(cfmap, annot = True, xticklabels = label, yticklabels = label)\n",
    "plt.xlabel(\"Prediction\")\n",
    "plt.title(\"Confusion Matrix for streetcar delay prediction (weighted)\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['loss', 'accuracy', 'accuracy_1']"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.metrics_names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Kaggle submission that was used as input for this notebook\n",
    "https://www.kaggle.com/knowledgegrappler/a-simple-nn-solution-with-keras-0-48611-pl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Summary\n",
    "This notebook shows methods for dealing with structured data in the context of a neural network.\n",
    "\n",
    "# Author\n",
    "\n",
    "Mark Ryan is a manager at Intact Insurance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "try_tf2",
   "language": "python",
   "name": "try_tf2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
